X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_ro%2Fvimconn_openstack.py;h=2b156d740540b7eee3b6a0ea43e98582e6049c9f;hb=0446cd5df24c38f95cea13b995c553e9b2403f21;hp=651afd3a41f8595f0726f4e06bdea4a77006927e;hpb=4d1ce22358bb9642b0a31488a31bdefbb595411b;p=osm%2FRO.git

diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py
index 651afd3a..2b156d74 100644
--- a/osm_ro/vimconn_openstack.py
+++ b/osm_ro/vimconn_openstack.py
@@ -32,11 +32,11 @@ to the VIM connector's SFC resources as follows:
 - Service Function (OSM) -> Port Pair Group (Neutron)
 - Service Function Path (OSM) -> Port Chain (Neutron)
 '''
-__author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C."
+__author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
 __date__  = "$22-sep-2017 23:59:59$"
 
 import vimconn
-# import json
+import json
 import logging
 import netaddr
 import time
@@ -44,6 +44,8 @@ import yaml
 import random
 import re
 import copy
+from pprint import pformat
+from types import StringTypes
 
 from novaclient import client as nClient, exceptions as nvExceptions
 from keystoneauth1.identity import v2, v3
@@ -52,7 +54,6 @@ import keystoneclient.exceptions as ksExceptions
 import keystoneclient.v3.client as ksClient_v3
 import keystoneclient.v2_0.client as ksClient_v2
 from glanceclient import client as glClient
-import glanceclient.client as gl1Client
 import glanceclient.exc as gl1Exceptions
 from  cinderclient import client as cClient
 from httplib import HTTPException
@@ -78,6 +79,18 @@ supportedClassificationTypes = ['legacy_flow_classifier']
 volume_timeout = 600
 server_timeout = 600
 
+
+class SafeDumper(yaml.SafeDumper):
+    def represent_data(self, data):
+        # Openstack APIs use custom subclasses of dict and YAML safe dumper
+        # is designed to not handle that (reference issue 142 of pyyaml)
+        if isinstance(data, dict) and data.__class__ != dict:
+            # A simple solution is to convert those items back to dicts
+            data = dict(data.items())
+
+        return super(SafeDumper, self).represent_data(data)
+
+
 class vimconnector(vimconn.vimconnector):
     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
                  log_level=None, config={}, persistent_info={}):
@@ -108,7 +121,6 @@ class vimconnector(vimconn.vimconnector):
             self.verify = False
         if self.config.get("ca_cert"):
             self.verify = self.config.get("ca_cert")
-        self.verify = self.config.get("insecure", False)
 
         if not url:
             raise TypeError('url param can not be NoneType')
@@ -119,7 +131,7 @@ class vimconnector(vimconn.vimconnector):
         self.neutron = self.session.get('neutron')
         self.cinder = self.session.get('cinder')
         self.glance = self.session.get('glance')
-        self.glancev1 = self.session.get('glancev1')
+        # self.glancev1 = self.session.get('glancev1')
         self.keystone = self.session.get('keystone')
         self.api_version3 = self.session.get('api_version3')
         self.vim_type = self.config.get("vim_type")
@@ -160,11 +172,30 @@ class vimconnector(vimconn.vimconnector):
             vimconn.vimconnector.__setitem__(self, index, value)
         self.session['reload_client'] = True
 
+    def serialize(self, value):
+        """Serialization of python basic types.
+
+        In the case value is not serializable a message will be logged and a
+        simple representation of the data that cannot be converted back to
+        python is returned.
+        """
+        if isinstance(value, StringTypes):
+            return value
+
+        try:
+            return yaml.dump(value, Dumper=SafeDumper,
+                             default_flow_style=True, width=256)
+        except yaml.representer.RepresenterError:
+                self.logger.debug(
+                    'The following entity cannot be serialized in YAML:'
+                    '\n\n%s\n\n', pformat(value), exc_info=True)
+                return str(value)
+
     def _reload_connection(self):
         '''Called before any operation, it check if credentials has changed
         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
         '''
-        #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) 
+        #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
         if self.session['reload_client']:
             if self.config.get('APIversion'):
                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
@@ -220,8 +251,8 @@ class vimconnector(vimconn.vimconnector):
                 glance_endpoint = None
             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
             #using version 1 of glance client in new_image()
-            self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
-                                                                       endpoint=glance_endpoint)
+            # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
+            #                                                            endpoint=glance_endpoint)
             self.session['reload_client'] = False
             self.persistent_info['session'] = self.session
             # add availablity zone info inside  self.persistent_info
@@ -464,7 +495,9 @@ class vimconnector(vimconn.vimconnector):
                     }
             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
             if ip_profile.get('gateway_address'):
-                subnet['gateway_ip'] = ip_profile.get('gateway_address')
+                subnet['gateway_ip'] = ip_profile['gateway_address']
+            else:
+                subnet['gateway_ip'] = None
             if ip_profile.get('dns_address'):
                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
             if 'dhcp_enabled' in ip_profile:
@@ -562,13 +595,13 @@ class vimconnector(vimconn.vimconnector):
                 net_id:         #VIM id of this network
                     status:     #Mandatory. Text with one of:
                                 #  DELETED (not found at vim)
-                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...) 
+                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
                                 #  OTHER (Vim reported other status not understood)
                                 #  ERROR (VIM indicates an ERROR status)
-                                #  ACTIVE, INACTIVE, DOWN (admin down), 
+                                #  ACTIVE, INACTIVE, DOWN (admin down),
                                 #  BUILD (on building process)
                                 #
-                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR 
+                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 
         '''
@@ -585,10 +618,9 @@ class vimconnector(vimconn.vimconnector):
 
                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
                     net['status'] = 'DOWN'
-                try:
-                    net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
-                except yaml.representer.RepresenterError:
-                    net['vim_info'] = str(net_vim)
+
+                net['vim_info'] = self.serialize(net_vim)
+
                 if net_vim.get('fault'):  #TODO
                     net['error_msg'] = str(net_vim['fault'])
             except vimconn.vimconnNotFoundException as e:
@@ -721,7 +753,7 @@ class vimconnector(vimconn.vimconnector):
                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 
-                #create flavor                 
+                #create flavor
                 new_flavor=self.nova.flavors.create(name,
                                 ram,
                                 vcpus,
@@ -771,40 +803,38 @@ class vimconnector(vimconn.vimconnector):
                 if "disk_format" in image_dict:
                     disk_format=image_dict["disk_format"]
                 else: #autodiscover based on extension
-                    if image_dict['location'][-6:]==".qcow2":
+                    if image_dict['location'].endswith(".qcow2"):
                         disk_format="qcow2"
-                    elif image_dict['location'][-4:]==".vhd":
+                    elif image_dict['location'].endswith(".vhd"):
                         disk_format="vhd"
-                    elif image_dict['location'][-5:]==".vmdk":
+                    elif image_dict['location'].endswith(".vmdk"):
                         disk_format="vmdk"
-                    elif image_dict['location'][-4:]==".vdi":
+                    elif image_dict['location'].endswith(".vdi"):
                         disk_format="vdi"
-                    elif image_dict['location'][-4:]==".iso":
+                    elif image_dict['location'].endswith(".iso"):
                         disk_format="iso"
-                    elif image_dict['location'][-4:]==".aki":
+                    elif image_dict['location'].endswith(".aki"):
                         disk_format="aki"
-                    elif image_dict['location'][-4:]==".ari":
+                    elif image_dict['location'].endswith(".ari"):
                         disk_format="ari"
-                    elif image_dict['location'][-4:]==".ami":
+                    elif image_dict['location'].endswith(".ami"):
                         disk_format="ami"
                     else:
                         disk_format="raw"
                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
-                if image_dict['location'][0:4]=="http":
-                    new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
-                            container_format="bare", location=image_dict['location'], disk_format=disk_format)
+                new_image = self.glance.images.create(name=image_dict['name'])
+                if image_dict['location'].startswith("http"):
+                    # TODO there is not a method to direct download. It must be downloaded locally with requests
+                    raise vimconn.vimconnNotImplemented("Cannot create image from URL")
                 else: #local path
                     with open(image_dict['location']) as fimage:
-                        new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
-                            container_format="bare", data=fimage, disk_format=disk_format)
-                #insert metadata. We cannot use 'new_image.properties.setdefault' 
-                #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
-                new_image_nova=self.nova.images.find(id=new_image.id)
-                new_image_nova.metadata.setdefault('location',image_dict['location'])
+                        self.glance.images.upload(new_image.id, fimage)
+                        #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
+                        #    container_format="bare", data=fimage, disk_format=disk_format)
                 metadata_to_load = image_dict.get('metadata')
-                if metadata_to_load:
-                    for k,v in yaml.load(metadata_to_load).iteritems():
-                        new_image_nova.metadata.setdefault(k,v)
+                #TODO location is a reserved word for current openstack versions. Use another word
+                metadata_to_load['location'] = image_dict['location']
+                self.glance.images.update(new_image.id, **metadata_to_load)
                 return new_image.id
             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
                 self._format_exception(e)
@@ -821,7 +851,7 @@ class vimconnector(vimconn.vimconnector):
         '''
         try:
             self._reload_connection()
-            self.nova.images.delete(image_id)
+            self.glance.images.delete(image_id)
             return image_id
         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
             self._format_exception(e)
@@ -830,7 +860,7 @@ class vimconnector(vimconn.vimconnector):
         '''Get the image id from image path in the VIM database. Returns the image_id'''
         try:
             self._reload_connection()
-            images = self.nova.images.list()
+            images = self.glance.images.list()
             for image in images:
                 if image.metadata.get("location")==path:
                     return image.id
@@ -853,17 +883,18 @@ class vimconnector(vimconn.vimconnector):
             self._reload_connection()
             filter_dict_os = filter_dict.copy()
             #First we filter by the available filter fields: name, id. The others are removed.
-            filter_dict_os.pop('checksum', None)
-            image_list = self.nova.images.findall(**filter_dict_os)
-            if len(image_list) == 0:
-                return []
-            #Then we filter by the rest of filter fields: checksum
+            image_list = self.glance.images.list()
             filtered_list = []
             for image in image_list:
                 try:
-                    image_class = self.glance.images.get(image.id)
-                    if 'checksum' not in filter_dict or image_class['checksum'] == filter_dict.get('checksum'):
-                        filtered_list.append(image_class.copy())
+                    if filter_dict.get("name") and image["name"] != filter_dict["name"]:
+                        continue
+                    if filter_dict.get("id") and image["id"] != filter_dict["id"]:
+                        continue
+                    if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
+                        continue
+
+                    filtered_list.append(image.copy())
                 except gl1Exceptions.HTTPNotFound:
                     pass
             return filtered_list
@@ -881,8 +912,8 @@ class vimconnector(vimconn.vimconnector):
                 return True
             if vm_status == 'ERROR':
                 return False
-            time.sleep(1)
-            elapsed_time += 1
+            time.sleep(5)
+            elapsed_time += 5
 
         # if we exceeded the timeout rollback
         if elapsed_time >= server_timeout:
@@ -980,6 +1011,7 @@ class vimconnector(vimconn.vimconnector):
             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
                 'size': (mandatory) string with the size of the disk in GB
+                'vim_id' (optional) should use this existing volume id
             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
                 availability_zone_index is None
@@ -1094,31 +1126,34 @@ class vimconnector(vimconn.vimconnector):
 
             # Create additional volumes in case these are present in disk_list
             base_disk_index = ord('b')
-            if disk_list != None:
+            if disk_list:
                 block_device_mapping = {}
                 for disk in disk_list:
-                    if 'image_id' in disk:
-                        volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
-                                    chr(base_disk_index), imageRef = disk['image_id'])
+                    if disk.get('vim_id'):
+                        block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
                     else:
-                        volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
-                                    chr(base_disk_index))
-                    created_items["volume:" + str(volume.id)] = True
-                    block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
+                        if 'image_id' in disk:
+                            volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
+                                                                chr(base_disk_index), imageRef=disk['image_id'])
+                        else:
+                            volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
+                                                                chr(base_disk_index))
+                        created_items["volume:" + str(volume.id)] = True
+                        block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
                     base_disk_index += 1
 
-                # Wait until volumes are with status available
-                keep_waiting = True
+                # Wait until created volumes are with status available
                 elapsed_time = 0
-                while keep_waiting and elapsed_time < volume_timeout:
-                    keep_waiting = False
-                    for volume_id in block_device_mapping.itervalues():
-                        if self.cinder.volumes.get(volume_id).status != 'available':
-                            keep_waiting = True
-                    if keep_waiting:
-                        time.sleep(1)
-                        elapsed_time += 1
-
+                while elapsed_time < volume_timeout:
+                    for created_item in created_items:
+                        v, _, volume_id = created_item.partition(":")
+                        if v == 'volume':
+                            if self.cinder.volumes.get(volume_id).status != 'available':
+                                break
+                    else:  # all ready: break from while
+                        break
+                    time.sleep(5)
+                    elapsed_time += 5
                 # If we exceeded the timeout rollback
                 if elapsed_time >= volume_timeout:
                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
@@ -1170,7 +1205,8 @@ class vimconnector(vimconn.vimconnector):
                                     continue
                             free_floating_ip = ip.get("floating_ip_address")
                         else:
-                            if isinstance(floating_network['floating_ip'], str):
+                            if isinstance(floating_network['floating_ip'], str) and \
+                                floating_network['floating_ip'].lower() != "true":
                                 pool_id = floating_network['floating_ip']
                             else:
                                 # Find the external network
@@ -1255,13 +1291,13 @@ class vimconnector(vimconn.vimconnector):
         Params:
             vm_id: uuid of the VM
             console_type, can be:
-                "novnc" (by default), "xvpvnc" for VNC types, 
+                "novnc" (by default), "xvpvnc" for VNC types,
                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
         Returns dict with the console parameters:
                 protocol: ssh, ftp, http, https, ...
-                server:   usually ip address 
-                port:     the http, ssh, ... port 
-                suffix:   extra text, e.g. the http path and query string   
+                server:   usually ip address
+                port:     the http, ssh, ... port
+                suffix:   extra text, e.g. the http path and query string
         '''
         self.logger.debug("Getting VM CONSOLE from VIM")
         try:
@@ -1361,14 +1397,14 @@ class vimconnector(vimconn.vimconnector):
                 vm_id:          #VIM id of this Virtual Machine
                     status:     #Mandatory. Text with one of:
                                 #  DELETED (not found at vim)
-                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...) 
+                                #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
                                 #  OTHER (Vim reported other status not understood)
                                 #  ERROR (VIM indicates an ERROR status)
-                                #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), 
+                                #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
                                 #  CREATING (on building process), ERROR
                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
                                 #
-                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR 
+                    error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
                     interfaces:
                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
@@ -1391,33 +1427,29 @@ class vimconnector(vimconn.vimconnector):
                 else:
                     vm['status']    = "OTHER"
                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
-                try:
-                    vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
-                except yaml.representer.RepresenterError:
-                    vm['vim_info'] = str(vm_vim)
+
+                vm['vim_info'] = self.serialize(vm_vim)
+
                 vm["interfaces"] = []
                 if vm_vim.get('fault'):
                     vm['error_msg'] = str(vm_vim['fault'])
                 #get interfaces
                 try:
                     self._reload_connection()
-                    port_dict=self.neutron.list_ports(device_id=vm_id)
+                    port_dict = self.neutron.list_ports(device_id=vm_id)
                     for port in port_dict["ports"]:
                         interface={}
-                        try:
-                            interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
-                        except yaml.representer.RepresenterError:
-                            interface['vim_info'] = str(port)
+                        interface['vim_info'] = self.serialize(port)
                         interface["mac_address"] = port.get("mac_address")
                         interface["vim_net_id"] = port["network_id"]
                         interface["vim_interface_id"] = port["id"]
-                        # check if OS-EXT-SRV-ATTR:host is there, 
+                        # check if OS-EXT-SRV-ATTR:host is there,
                         # in case of non-admin credentials, it will be missing
                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
                         interface["pci"] = None
 
-                        # check if binding:profile is there, 
+                        # check if binding:profile is there,
                         # in case of non-admin credentials, it will be missing
                         if port.get('binding:profile'):
                             if port['binding:profile'].get('pci_slot'):
@@ -1435,16 +1467,20 @@ class vimconnector(vimconn.vimconnector):
                             interface["vlan"] = network['network'].get('provider:segmentation_id')
                         ips=[]
                         #look for floating ip address
-                        floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
-                        if floating_ip_dict.get("floatingips"):
-                            ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
+                        try:
+                            floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
+                            if floating_ip_dict.get("floatingips"):
+                                ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
+                        except Exception:
+                            pass
 
                         for subnet in port["fixed_ips"]:
                             ips.append(subnet["ip_address"])
                         interface["ip_address"] = ";".join(ips)
                         vm["interfaces"].append(interface)
                 except Exception as e:
-                    self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
+                    self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
+                                      exc_info=True)
             except vimconn.vimconnNotFoundException as e:
                 self.logger.error("Exception getting vm status: %s", str(e))
                 vm['status'] = "DELETED"
@@ -1611,7 +1647,7 @@ class vimconnector(vimconn.vimconnector):
         self.logger.debug("osconnector: Adding a new user to VIM")
         try:
             self._reload_connection()
-            user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
+            user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
             return user.id
         except ksExceptions.ConnectionError as e: