X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_ro%2Fvimconn_openstack.py;h=b698eda3f1e4c7eea7902e27a7c6d56c616db8a9;hb=a05b65a809f3680d38f162b1e980e4258b0e37c2;hp=57e22575d4dae9baee765a67d09b78bfa183053d;hpb=c76a3eee8272c5f3c66f43b8eafe2c48273ee0c7;p=osm%2FRO.git diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py index 57e22575..b698eda3 100644 --- a/osm_ro/vimconn_openstack.py +++ b/osm_ro/vimconn_openstack.py @@ -44,6 +44,8 @@ import yaml import random import re import copy +from pprint import pformat +from types import StringTypes from novaclient import client as nClient, exceptions as nvExceptions from keystoneauth1.identity import v2, v3 @@ -77,6 +79,18 @@ supportedClassificationTypes = ['legacy_flow_classifier'] volume_timeout = 600 server_timeout = 600 + +class SafeDumper(yaml.SafeDumper): + def represent_data(self, data): + # Openstack APIs use custom subclasses of dict and YAML safe dumper + # is designed to not handle that (reference issue 142 of pyyaml) + if isinstance(data, dict) and data.__class__ != dict: + # A simple solution is to convert those items back to dicts + data = dict(data.items()) + + return super(SafeDumper, self).represent_data(data) + + class vimconnector(vimconn.vimconnector): def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}): @@ -113,6 +127,7 @@ class vimconnector(vimconn.vimconnector): self.persistent_info = persistent_info self.availability_zone = persistent_info.get('availability_zone', None) self.session = persistent_info.get('session', {'reload_client': True}) + self.my_tenant_id = self.session.get('my_tenant_id') self.nova = self.session.get('nova') self.neutron = self.session.get('neutron') self.cinder = self.session.get('cinder') @@ -130,6 +145,11 @@ class vimconnector(vimconn.vimconnector): self.logger = logging.getLogger('openmano.vim.openstack') + # allow security_groups to be a list or a single string + if isinstance(self.config.get('security_groups'), str): + self.config['security_groups'] = [self.config['security_groups']] + self.security_groups_id = None + ####### VIO Specific Changes ######### if self.vim_type == "VIO": self.logger = logging.getLogger('openmano.vim.vio') @@ -158,11 +178,30 @@ class vimconnector(vimconn.vimconnector): vimconn.vimconnector.__setitem__(self, index, value) self.session['reload_client'] = True + def serialize(self, value): + """Serialization of python basic types. + + In the case value is not serializable a message will be logged and a + simple representation of the data that cannot be converted back to + python is returned. + """ + if isinstance(value, StringTypes): + return value + + try: + return yaml.dump(value, Dumper=SafeDumper, + default_flow_style=True, width=256) + except yaml.representer.RepresenterError: + self.logger.debug( + 'The following entity cannot be serialized in YAML:' + '\n\n%s\n\n', pformat(value), exc_info=True) + return str(value) + def _reload_connection(self): '''Called before any operation, it check if credentials has changed Throw keystoneclient.apiclient.exceptions.AuthorizationFailure ''' - #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) + #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) if self.session['reload_client']: if self.config.get('APIversion'): self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3' @@ -194,8 +233,10 @@ class vimconnector(vimconn.vimconnector): tenant_name=self.tenant_name, tenant_id=self.tenant_id) sess = session.Session(auth=auth, verify=self.verify) + # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River Titanium cloud and StarlingX + region_name = self.config.get('region_name') if self.api_version3: - self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type) + self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type, region_name=region_name) else: self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type) self.session['keystone'] = self.keystone @@ -208,16 +249,21 @@ class vimconnector(vimconn.vimconnector): version = self.config.get("microversion") if not version: version = "2.1" - self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type) - self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type) - self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type) + # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River Titanium cloud and StarlingX + self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + try: + self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id() + except Exception as e: + self.logger.error("Cannot get project_id from session", exc_info=True) if self.endpoint_type == "internalURL": glance_service_id = self.keystone.services.list(name="glance")[0].id glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url else: glance_endpoint = None self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint) - #using version 1 of glance client in new_image() + # using version 1 of glance client in new_image() # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess, # endpoint=glance_endpoint) self.session['reload_client'] = False @@ -225,6 +271,7 @@ class vimconnector(vimconn.vimconnector): # add availablity zone info inside self.persistent_info self._set_availablity_zones() self.persistent_info['availability_zone'] = self.availability_zone + self.security_groups_id = None # force to get again security_groups_ids next time they are needed def __net_os2mano(self, net_list_dict): '''Transform the net openstack format to mano format @@ -347,13 +394,12 @@ class vimconnector(vimconn.vimconnector): def _format_exception(self, exception): '''Transform a keystone, nova, neutron exception into a vimconn exception''' - if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, - ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed - )): - raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) - elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)): + if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, gl1Exceptions.HTTPNotFound)): raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception)) - elif isinstance(exception, (KeyError, nvExceptions.BadRequest)): + elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, + ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)): + raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) + elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)): raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception)) elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException, neExceptions.NeutronException)): @@ -366,6 +412,30 @@ class vimconnector(vimconn.vimconnector): self.logger.error("General Exception " + str(exception), exc_info=True) raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) + def _get_ids_from_name(self): + """ + Obtain ids from name of tenant and security_groups. Store at self .security_groups_id" + :return: None + """ + # get tenant_id if only tenant_name is supplied + self._reload_connection() + if not self.my_tenant_id: + raise vimconn.vimconnConnectionException("Error getting tenant information from name={} id={}". + format(self.tenant_name, self.tenant_id)) + if self.config.get('security_groups') and not self.security_groups_id: + # convert from name to id + neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"] + + self.security_groups_id = [] + for sg in self.config.get('security_groups'): + for neutron_sg in neutron_sg_list: + if sg in (neutron_sg["id"], neutron_sg["name"]): + self.security_groups_id.append(neutron_sg["id"]) + break + else: + self.security_groups_id = None + raise vimconn.vimconnConnectionException("Not found security group {} for this tenant".format(sg)) + def get_tenant_list(self, filter_dict={}): '''Obtain tenants of VIM filter_dict can contain the following keys: @@ -401,7 +471,7 @@ class vimconnector(vimconn.vimconnector): else: project = self.keystone.tenants.create(tenant_name, tenant_description) return project.id - except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e: + except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError) as e: self._format_exception(e) def delete_tenant(self, tenant_id): @@ -414,7 +484,7 @@ class vimconnector(vimconn.vimconnector): else: self.keystone.tenants.delete(tenant_id) return tenant_id - except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e: + except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError) as e: self._format_exception(e) def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None): @@ -536,7 +606,9 @@ class vimconnector(vimconn.vimconnector): subnets.append(subnet) net["subnets"] = subnets net["encapsulation"] = net.get('provider:network_type') + net["encapsulation_type"] = net.get('provider:network_type') net["segmentation_id"] = net.get('provider:segmentation_id') + net["encapsulation_id"] = net.get('provider:segmentation_id') return net def delete_network(self, net_id): @@ -564,13 +636,13 @@ class vimconnector(vimconn.vimconnector): net_id: #VIM id of this network status: #Mandatory. Text with one of: # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) + # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) # OTHER (Vim reported other status not understood) # ERROR (VIM indicates an ERROR status) - # ACTIVE, INACTIVE, DOWN (admin down), + # ACTIVE, INACTIVE, DOWN (admin down), # BUILD (on building process) # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR vim_info: #Text with plain information obtained from vim (yaml.safe_dump) ''' @@ -587,10 +659,9 @@ class vimconnector(vimconn.vimconnector): if net['status'] == "ACTIVE" and not net_vim['admin_state_up']: net['status'] = 'DOWN' - try: - net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256) - except yaml.representer.RepresenterError: - net['vim_info'] = str(net_vim) + + net['vim_info'] = self.serialize(net_vim) + if net_vim.get('fault'): #TODO net['error_msg'] = str(net_vim['fault']) except vimconn.vimconnNotFoundException as e: @@ -793,7 +864,14 @@ class vimconnector(vimconn.vimconnector): else: disk_format="raw" self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location']) - new_image = self.glance.images.create(name=image_dict['name']) + if self.vim_type == "VIO": + container_format = "bare" + if 'container_format' in image_dict: + container_format = image_dict['container_format'] + new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format, + disk_format=disk_format) + else: + new_image = self.glance.images.create(name=image_dict['name']) if image_dict['location'].startswith("http"): # TODO there is not a method to direct download. It must be downloaded locally with requests raise vimconn.vimconnNotImplemented("Cannot create image from URL") @@ -803,8 +881,11 @@ class vimconnector(vimconn.vimconnector): #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes", # container_format="bare", data=fimage, disk_format=disk_format) metadata_to_load = image_dict.get('metadata') - #TODO location is a reserved word for current openstack versions. Use another word - metadata_to_load['location'] = image_dict['location'] + # TODO location is a reserved word for current openstack versions. fixed for VIO please check for openstack + if self.vim_type == "VIO": + metadata_to_load['upload_location'] = image_dict['location'] + else: + metadata_to_load['location'] = image_dict['location'] self.glance.images.update(new_image.id, **metadata_to_load) return new_image.id except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e: @@ -824,7 +905,7 @@ class vimconnector(vimconn.vimconnector): self._reload_connection() self.glance.images.delete(image_id) return image_id - except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove + except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e: #TODO remove self._format_exception(e) def get_image_id_from_path(self, path): @@ -1004,15 +1085,22 @@ class vimconnector(vimconn.vimconnector): self._reload_connection() # metadata_vpci = {} # For a specific neutron plugin block_device_mapping = None + for net in net_list: if not net.get("net_id"): # skip non connected iface continue - port_dict={ + port_dict = { "network_id": net["net_id"], "name": net.get("name"), "admin_state_up": True } + if self.config.get("security_groups") and net.get("port_security") is not False and \ + not self.config.get("no_port_security_extension"): + if not self.security_groups_id: + self._get_ids_from_name() + port_dict["security_groups"] = self.security_groups_id + if net["type"]=="virtual": pass # if "vpci" in net: @@ -1089,9 +1177,6 @@ class vimconnector(vimconn.vimconnector): self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'", name, image_id, flavor_id, str(net_list_vim), description) - security_groups = self.config.get('security_groups') - if type(security_groups) is str: - security_groups = ( security_groups, ) # cloud config config_drive, userdata = self._create_user_data(cloud_config) @@ -1135,10 +1220,12 @@ class vimconnector(vimconn.vimconnector): self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, " "availability_zone={}, key_name={}, userdata={}, config_drive={}, " "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim, - security_groups, vm_av_zone, self.config.get('keypair'), - userdata, config_drive, block_device_mapping)) + self.config.get("security_groups"), vm_av_zone, + self.config.get('keypair'), userdata, config_drive, + block_device_mapping)) server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, - security_groups=security_groups, + security_groups=self.config.get("security_groups"), + # TODO remove security_groups in future versions. Already at neutron port availability_zone=vm_av_zone, key_name=self.config.get('keypair'), userdata=userdata, @@ -1262,13 +1349,13 @@ class vimconnector(vimconn.vimconnector): Params: vm_id: uuid of the VM console_type, can be: - "novnc" (by default), "xvpvnc" for VNC types, + "novnc" (by default), "xvpvnc" for VNC types, "rdp-html5" for RDP types, "spice-html5" for SPICE types Returns dict with the console parameters: protocol: ssh, ftp, http, https, ... - server: usually ip address - port: the http, ssh, ... port - suffix: extra text, e.g. the http path and query string + server: usually ip address + port: the http, ssh, ... port + suffix: extra text, e.g. the http path and query string ''' self.logger.debug("Getting VM CONSOLE from VIM") try: @@ -1368,14 +1455,14 @@ class vimconnector(vimconn.vimconnector): vm_id: #VIM id of this Virtual Machine status: #Mandatory. Text with one of: # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) + # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) # OTHER (Vim reported other status not understood) # ERROR (VIM indicates an ERROR status) - # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), + # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), # CREATING (on building process), ERROR # ACTIVE:NoMgmtIP (Active but any of its interface has an IP address # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR vim_info: #Text with plain information obtained from vim (yaml.safe_dump) interfaces: - vim_info: #Text with plain information obtained from vim (yaml.safe_dump) @@ -1398,10 +1485,9 @@ class vimconnector(vimconn.vimconnector): else: vm['status'] = "OTHER" vm['error_msg'] = "VIM status reported " + vm_vim['status'] - try: - vm['vim_info'] = yaml.safe_dump(vm_vim, default_flow_style=True, width=256) - except yaml.representer.RepresenterError: - vm['vim_info'] = str(vm_vim) + + vm['vim_info'] = self.serialize(vm_vim) + vm["interfaces"] = [] if vm_vim.get('fault'): vm['error_msg'] = str(vm_vim['fault']) @@ -1411,20 +1497,17 @@ class vimconnector(vimconn.vimconnector): port_dict = self.neutron.list_ports(device_id=vm_id) for port in port_dict["ports"]: interface={} - try: - interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256) - except yaml.representer.RepresenterError: - interface['vim_info'] = str(port) + interface['vim_info'] = self.serialize(port) interface["mac_address"] = port.get("mac_address") interface["vim_net_id"] = port["network_id"] interface["vim_interface_id"] = port["id"] - # check if OS-EXT-SRV-ATTR:host is there, + # check if OS-EXT-SRV-ATTR:host is there, # in case of non-admin credentials, it will be missing if vm_vim.get('OS-EXT-SRV-ATTR:host'): interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host'] interface["pci"] = None - # check if binding:profile is there, + # check if binding:profile is there, # in case of non-admin credentials, it will be missing if port.get('binding:profile'): if port['binding:profile'].get('pci_slot'): @@ -1937,7 +2020,7 @@ class vimconnector(vimconn.vimconnector): self._reload_connection() # In networking-sfc the MPLS encapsulation is legacy # should be used when no full SFC Encapsulation is intended - sfc_encap = 'mpls' + correlation = 'mpls' if sfc_encap: correlation = 'nsh' sfp_dict = {'name': name,