Added support for region_name for Distributed cloud implementation of WindRiver Titan...
[osm/RO.git] / osm_ro / vimconn_openstack.py
index 975abaf..a24de00 100644 (file)
@@ -44,6 +44,8 @@ import yaml
 import random
 import re
 import copy
+from pprint import pformat
+from types import StringTypes
 
 from novaclient import client as nClient, exceptions as nvExceptions
 from keystoneauth1.identity import v2, v3
@@ -77,6 +79,18 @@ supportedClassificationTypes = ['legacy_flow_classifier']
 volume_timeout = 600
 server_timeout = 600
 
+
+class SafeDumper(yaml.SafeDumper):
+    def represent_data(self, data):
+        # Openstack APIs use custom subclasses of dict and YAML safe dumper
+        # is designed to not handle that (reference issue 142 of pyyaml)
+        if isinstance(data, dict) and data.__class__ != dict:
+            # A simple solution is to convert those items back to dicts
+            data = dict(data.items())
+
+        return super(SafeDumper, self).represent_data(data)
+
+
 class vimconnector(vimconn.vimconnector):
     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
                  log_level=None, config={}, persistent_info={}):
@@ -158,11 +172,30 @@ class vimconnector(vimconn.vimconnector):
             vimconn.vimconnector.__setitem__(self, index, value)
         self.session['reload_client'] = True
 
+    def serialize(self, value):
+        """Serialization of python basic types.
+
+        In the case value is not serializable a message will be logged and a
+        simple representation of the data that cannot be converted back to
+        python is returned.
+        """
+        if isinstance(value, StringTypes):
+            return value
+
+        try:
+            return yaml.dump(value, Dumper=SafeDumper,
+                             default_flow_style=True, width=256)
+        except yaml.representer.RepresenterError:
+                self.logger.debug(
+                    'The following entity cannot be serialized in YAML:'
+                    '\n\n%s\n\n', pformat(value), exc_info=True)
+                return str(value)
+
     def _reload_connection(self):
         '''Called before any operation, it check if credentials has changed
         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
         '''
-        #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) 
+        #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
         if self.session['reload_client']:
             if self.config.get('APIversion'):
                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
@@ -195,7 +228,10 @@ class vimconnector(vimconn.vimconnector):
                                    tenant_id=self.tenant_id)
             sess = session.Session(auth=auth, verify=self.verify)
             if self.api_version3:
-                self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
+                if self.config.get('region_name'):
+                    self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type, region_name=self.config['region_name'])
+                else:
+                    self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
             else:
                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
             self.session['keystone'] = self.keystone
@@ -208,9 +244,15 @@ class vimconnector(vimconn.vimconnector):
             version = self.config.get("microversion")
             if not version:
                 version = "2.1"
-            self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
-            self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
-            self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
+            # Added region to support Distributed cloud as implemented in WindRiver Titanium Cloud and StarlingX
+            if self.config.get('region_name'):
+                self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type, region_name=self.config['region_name'])
+                self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type, region_name=self.config['region_name'])
+                self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type, region_name=self.config['region_name'])
+            else:
+                self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
+                self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
+                self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
             if self.endpoint_type == "internalURL":
                 glance_service_id = self.keystone.services.list(name="glance")[0].id
                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
@@ -347,15 +389,16 @@ class vimconnector(vimconn.vimconnector):
 
     def _format_exception(self, exception):
         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
-        if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
-                                  ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
-                                  )):
+        if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, gl1Exceptions.HTTPNotFound)):
+            raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
+        elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
+                               ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
+        elif isinstance(exception,  (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
+            raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception))
         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
-                                    neExceptions.NeutronException, nvExceptions.BadRequest)):
+                                    neExceptions.NeutronException)):
             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
-        elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
-            raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
         elif isinstance(exception, nvExceptions.Conflict):
             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
         elif isinstance(exception, vimconn.vimconnException):
@@ -399,7 +442,7 @@ class vimconnector(vimconn.vimconnector):
             else:
                 project = self.keystone.tenants.create(tenant_name, tenant_description)
             return project.id
-        except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
+        except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)  as e:
             self._format_exception(e)
 
     def delete_tenant(self, tenant_id):
@@ -412,7 +455,7 @@ class vimconnector(vimconn.vimconnector):
             else:
                 self.keystone.tenants.delete(tenant_id)
             return tenant_id
-        except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
+        except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)  as e:
             self._format_exception(e)
 
     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
@@ -534,7 +577,9 @@ class vimconnector(vimconn.vimconnector):
             subnets.append(subnet)
         net["subnets"] = subnets
         net["encapsulation"] = net.get('provider:network_type')
+        net["encapsulation_type"] = net.get('provider:network_type')
         net["segmentation_id"] = net.get('provider:segmentation_id')
+        net["encapsulation_id"] = net.get('provider:segmentation_id')
         return net
 
     def delete_network(self, net_id):
@@ -562,13 +607,13 @@ class vimconnector(vimconn.vimconnector):
                 net_id:         #VIM id of this network
                     status:     #Mandatory. Text with one of:
                                 #  DELETED (not found at vim)
-                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...) 
+                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
                                 #  OTHER (Vim reported other status not understood)
                                 #  ERROR (VIM indicates an ERROR status)
-                                #  ACTIVE, INACTIVE, DOWN (admin down), 
+                                #  ACTIVE, INACTIVE, DOWN (admin down),
                                 #  BUILD (on building process)
                                 #
-                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR 
+                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 
         '''
@@ -585,10 +630,9 @@ class vimconnector(vimconn.vimconnector):
 
                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
                     net['status'] = 'DOWN'
-                try:
-                    net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
-                except yaml.representer.RepresenterError:
-                    net['vim_info'] = str(net_vim)
+
+                net['vim_info'] = self.serialize(net_vim)
+
                 if net_vim.get('fault'):  #TODO
                     net['error_msg'] = str(net_vim['fault'])
             except vimconn.vimconnNotFoundException as e:
@@ -662,83 +706,84 @@ class vimconnector(vimconn.vimconnector):
         retry=0
         max_retries=3
         name_suffix = 0
-        name=flavor_data['name']
-        while retry<max_retries:
-            retry+=1
-            try:
-                self._reload_connection()
-                if change_name_if_used:
-                    #get used names
-                    fl_names=[]
-                    fl=self.nova.flavors.list()
-                    for f in fl:
-                        fl_names.append(f.name)
-                    while name in fl_names:
-                        name_suffix += 1
-                        name = flavor_data['name']+"-" + str(name_suffix)
-
-                ram = flavor_data.get('ram',64)
-                vcpus = flavor_data.get('vcpus',1)
-                numa_properties=None
-
-                extended = flavor_data.get("extended")
-                if extended:
-                    numas=extended.get("numas")
-                    if numas:
-                        numa_nodes = len(numas)
-                        if numa_nodes > 1:
-                            return -1, "Can not add flavor with more than one numa"
-                        numa_properties = {"hw:numa_nodes":str(numa_nodes)}
-                        numa_properties["hw:mem_page_size"] = "large"
-                        numa_properties["hw:cpu_policy"] = "dedicated"
-                        numa_properties["hw:numa_mempolicy"] = "strict"
-                        if self.vim_type == "VIO":
-                            numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
-                            numa_properties["vmware:latency_sensitivity_level"] = "high"
-                        for numa in numas:
-                            #overwrite ram and vcpus
-                            #check if key 'memory' is present in numa else use ram value at flavor
-                            if 'memory' in numa:
-                                ram = numa['memory']*1024
-                            #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
-                            if 'paired-threads' in numa:
-                                vcpus = numa['paired-threads']*2
-                                #cpu_thread_policy "require" implies that the compute node must have an STM architecture
-                                numa_properties["hw:cpu_thread_policy"] = "require"
-                                numa_properties["hw:cpu_policy"] = "dedicated"
-                            elif 'cores' in numa:
-                                vcpus = numa['cores']
-                                # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
-                                numa_properties["hw:cpu_thread_policy"] = "isolate"
-                                numa_properties["hw:cpu_policy"] = "dedicated"
-                            elif 'threads' in numa:
-                                vcpus = numa['threads']
-                                # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
-                                numa_properties["hw:cpu_thread_policy"] = "prefer"
-                                numa_properties["hw:cpu_policy"] = "dedicated"
-                            # for interface in numa.get("interfaces",() ):
-                            #     if interface["dedicated"]=="yes":
-                            #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
-                            #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
-
-                #create flavor                 
-                new_flavor=self.nova.flavors.create(name,
-                                ram,
-                                vcpus,
-                                flavor_data.get('disk',0),
-                                is_public=flavor_data.get('is_public', True)
-                            )
-                #add metadata
-                if numa_properties:
-                    new_flavor.set_keys(numa_properties)
-                return new_flavor.id
-            except nvExceptions.Conflict as e:
-                if change_name_if_used and retry < max_retries:
-                    continue
-                self._format_exception(e)
-            #except nvExceptions.BadRequest as e:
-            except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
-                self._format_exception(e)
+        try:
+            name=flavor_data['name']
+            while retry<max_retries:
+                retry+=1
+                try:
+                    self._reload_connection()
+                    if change_name_if_used:
+                        #get used names
+                        fl_names=[]
+                        fl=self.nova.flavors.list()
+                        for f in fl:
+                            fl_names.append(f.name)
+                        while name in fl_names:
+                            name_suffix += 1
+                            name = flavor_data['name']+"-" + str(name_suffix)
+
+                    ram = flavor_data.get('ram',64)
+                    vcpus = flavor_data.get('vcpus',1)
+                    numa_properties=None
+
+                    extended = flavor_data.get("extended")
+                    if extended:
+                        numas=extended.get("numas")
+                        if numas:
+                            numa_nodes = len(numas)
+                            if numa_nodes > 1:
+                                return -1, "Can not add flavor with more than one numa"
+                            numa_properties = {"hw:numa_nodes":str(numa_nodes)}
+                            numa_properties["hw:mem_page_size"] = "large"
+                            numa_properties["hw:cpu_policy"] = "dedicated"
+                            numa_properties["hw:numa_mempolicy"] = "strict"
+                            if self.vim_type == "VIO":
+                                numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
+                                numa_properties["vmware:latency_sensitivity_level"] = "high"
+                            for numa in numas:
+                                #overwrite ram and vcpus
+                                #check if key 'memory' is present in numa else use ram value at flavor
+                                if 'memory' in numa:
+                                    ram = numa['memory']*1024
+                                #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
+                                if 'paired-threads' in numa:
+                                    vcpus = numa['paired-threads']*2
+                                    #cpu_thread_policy "require" implies that the compute node must have an STM architecture
+                                    numa_properties["hw:cpu_thread_policy"] = "require"
+                                    numa_properties["hw:cpu_policy"] = "dedicated"
+                                elif 'cores' in numa:
+                                    vcpus = numa['cores']
+                                    # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
+                                    numa_properties["hw:cpu_thread_policy"] = "isolate"
+                                    numa_properties["hw:cpu_policy"] = "dedicated"
+                                elif 'threads' in numa:
+                                    vcpus = numa['threads']
+                                    # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
+                                    numa_properties["hw:cpu_thread_policy"] = "prefer"
+                                    numa_properties["hw:cpu_policy"] = "dedicated"
+                                # for interface in numa.get("interfaces",() ):
+                                #     if interface["dedicated"]=="yes":
+                                #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
+                                #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
+
+                    #create flavor
+                    new_flavor=self.nova.flavors.create(name,
+                                    ram,
+                                    vcpus,
+                                    flavor_data.get('disk',0),
+                                    is_public=flavor_data.get('is_public', True)
+                                )
+                    #add metadata
+                    if numa_properties:
+                        new_flavor.set_keys(numa_properties)
+                    return new_flavor.id
+                except nvExceptions.Conflict as e:
+                    if change_name_if_used and retry < max_retries:
+                        continue
+                    self._format_exception(e)
+        #except nvExceptions.BadRequest as e:
+        except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
+            self._format_exception(e)
 
     def delete_flavor(self,flavor_id):
         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
@@ -790,7 +835,14 @@ class vimconnector(vimconn.vimconnector):
                     else:
                         disk_format="raw"
                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
-                new_image = self.glance.images.create(name=image_dict['name'])
+                if self.vim_type == "VIO":
+                    container_format = "bare"
+                    if 'container_format' in image_dict:
+                        container_format = image_dict['container_format']
+                    new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
+                                                          disk_format=disk_format)
+                else:
+                    new_image = self.glance.images.create(name=image_dict['name'])
                 if image_dict['location'].startswith("http"):
                     # TODO there is not a method to direct download. It must be downloaded locally with requests
                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
@@ -800,8 +852,11 @@ class vimconnector(vimconn.vimconnector):
                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
                         #    container_format="bare", data=fimage, disk_format=disk_format)
                 metadata_to_load = image_dict.get('metadata')
-                #TODO location is a reserved word for current openstack versions. Use another word
-                metadata_to_load['location'] = image_dict['location']
+                # TODO location is a reserved word for current openstack versions. fixed for VIO please check for openstack
+                if self.vim_type == "VIO":
+                    metadata_to_load['upload_location'] = image_dict['location']
+                else:
+                    metadata_to_load['location'] = image_dict['location']
                 self.glance.images.update(new_image.id, **metadata_to_load)
                 return new_image.id
             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
@@ -821,7 +876,7 @@ class vimconnector(vimconn.vimconnector):
             self._reload_connection()
             self.glance.images.delete(image_id)
             return image_id
-        except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
+        except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e: #TODO remove
             self._format_exception(e)
 
     def get_image_id_from_path(self, path):
@@ -1259,13 +1314,13 @@ class vimconnector(vimconn.vimconnector):
         Params:
             vm_id: uuid of the VM
             console_type, can be:
-                "novnc" (by default), "xvpvnc" for VNC types, 
+                "novnc" (by default), "xvpvnc" for VNC types,
                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
         Returns dict with the console parameters:
                 protocol: ssh, ftp, http, https, ...
-                server:   usually ip address 
-                port:     the http, ssh, ... port 
-                suffix:   extra text, e.g. the http path and query string   
+                server:   usually ip address
+                port:     the http, ssh, ... port
+                suffix:   extra text, e.g. the http path and query string
         '''
         self.logger.debug("Getting VM CONSOLE from VIM")
         try:
@@ -1365,14 +1420,14 @@ class vimconnector(vimconn.vimconnector):
                 vm_id:          #VIM id of this Virtual Machine
                     status:     #Mandatory. Text with one of:
                                 #  DELETED (not found at vim)
-                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...) 
+                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
                                 #  OTHER (Vim reported other status not understood)
                                 #  ERROR (VIM indicates an ERROR status)
-                                #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), 
+                                #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
                                 #  CREATING (on building process), ERROR
                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
                                 #
-                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR 
+                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
                     interfaces:
                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
@@ -1395,10 +1450,9 @@ class vimconnector(vimconn.vimconnector):
                 else:
                     vm['status']    = "OTHER"
                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
-                try:
-                    vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
-                except yaml.representer.RepresenterError:
-                    vm['vim_info'] = str(vm_vim)
+
+                vm['vim_info'] = self.serialize(vm_vim)
+
                 vm["interfaces"] = []
                 if vm_vim.get('fault'):
                     vm['error_msg'] = str(vm_vim['fault'])
@@ -1408,20 +1462,17 @@ class vimconnector(vimconn.vimconnector):
                     port_dict = self.neutron.list_ports(device_id=vm_id)
                     for port in port_dict["ports"]:
                         interface={}
-                        try:
-                            interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
-                        except yaml.representer.RepresenterError:
-                            interface['vim_info'] = str(port)
+                        interface['vim_info'] = self.serialize(port)
                         interface["mac_address"] = port.get("mac_address")
                         interface["vim_net_id"] = port["network_id"]
                         interface["vim_interface_id"] = port["id"]
-                        # check if OS-EXT-SRV-ATTR:host is there, 
+                        # check if OS-EXT-SRV-ATTR:host is there,
                         # in case of non-admin credentials, it will be missing
                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
                         interface["pci"] = None
 
-                        # check if binding:profile is there, 
+                        # check if binding:profile is there,
                         # in case of non-admin credentials, it will be missing
                         if port.get('binding:profile'):
                             if port['binding:profile'].get('pci_slot'):
@@ -1934,7 +1985,7 @@ class vimconnector(vimconn.vimconnector):
             self._reload_connection()
             # In networking-sfc the MPLS encapsulation is legacy
             # should be used when no full SFC Encapsulation is intended
-            sfc_encap = 'mpls'
+            correlation = 'mpls'
             if sfc_encap:
                 correlation = 'nsh'
             sfp_dict = {'name': name,