fixing bug: vim config were not loaded at vim_thread
[osm/RO.git] / osm_ro / vimconn_openstack.py
index 55f910b..0d2603f 100644 (file)
@@ -101,7 +101,14 @@ class vimconnector(vimconn.vimconnector):
         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
                                       config)
 
-        self.insecure = self.config.get("insecure", False)
+        if self.config.get("insecure") and self.config.get("ca_cert"):
+            raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
+        self.verify = True
+        if self.config.get("insecure"):
+            self.verify = False
+        if self.config.get("ca_cert"):
+            self.verify = self.config.get("ca_cert")
+
         if not url:
             raise TypeError('url param can not be NoneType')
         self.persistent_info = persistent_info
@@ -111,7 +118,7 @@ class vimconnector(vimconn.vimconnector):
         self.neutron = self.session.get('neutron')
         self.cinder = self.session.get('cinder')
         self.glance = self.session.get('glance')
-        self.glancev1 = self.session.get('glancev1')
+        self.glancev1 = self.session.get('glancev1')
         self.keystone = self.session.get('keystone')
         self.api_version3 = self.session.get('api_version3')
         self.vim_type = self.config.get("vim_type")
@@ -187,7 +194,7 @@ class vimconnector(vimconn.vimconnector):
                                    password=self.passwd,
                                    tenant_name=self.tenant_name,
                                    tenant_id=self.tenant_id)
-            sess = session.Session(auth=auth, verify=not self.insecure)
+            sess = session.Session(auth=auth, verify=self.verify)
             if self.api_version3:
                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
             else:
@@ -212,8 +219,8 @@ class vimconnector(vimconn.vimconnector):
                 glance_endpoint = None
             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
             #using version 1 of glance client in new_image()
-            self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
-                                                                       endpoint=glance_endpoint)
+            self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
+                                                                       endpoint=glance_endpoint)
             self.session['reload_client'] = False
             self.persistent_info['session'] = self.session
             # add availablity zone info inside  self.persistent_info
@@ -456,7 +463,9 @@ class vimconnector(vimconn.vimconnector):
                     }
             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
             if ip_profile.get('gateway_address'):
-                subnet['gateway_ip'] = ip_profile.get('gateway_address')
+                subnet['gateway_ip'] = ip_profile['gateway_address']
+            else:
+                subnet['gateway_ip'] = None
             if ip_profile.get('dns_address'):
                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
             if 'dhcp_enabled' in ip_profile:
@@ -763,40 +772,38 @@ class vimconnector(vimconn.vimconnector):
                 if "disk_format" in image_dict:
                     disk_format=image_dict["disk_format"]
                 else: #autodiscover based on extension
-                    if image_dict['location'][-6:]==".qcow2":
+                    if image_dict['location'].endswith(".qcow2"):
                         disk_format="qcow2"
-                    elif image_dict['location'][-4:]==".vhd":
+                    elif image_dict['location'].endswith(".vhd"):
                         disk_format="vhd"
-                    elif image_dict['location'][-5:]==".vmdk":
+                    elif image_dict['location'].endswith(".vmdk"):
                         disk_format="vmdk"
-                    elif image_dict['location'][-4:]==".vdi":
+                    elif image_dict['location'].endswith(".vdi"):
                         disk_format="vdi"
-                    elif image_dict['location'][-4:]==".iso":
+                    elif image_dict['location'].endswith(".iso"):
                         disk_format="iso"
-                    elif image_dict['location'][-4:]==".aki":
+                    elif image_dict['location'].endswith(".aki"):
                         disk_format="aki"
-                    elif image_dict['location'][-4:]==".ari":
+                    elif image_dict['location'].endswith(".ari"):
                         disk_format="ari"
-                    elif image_dict['location'][-4:]==".ami":
+                    elif image_dict['location'].endswith(".ami"):
                         disk_format="ami"
                     else:
                         disk_format="raw"
                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
-                if image_dict['location'][0:4]=="http":
-                    new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
-                            container_format="bare", location=image_dict['location'], disk_format=disk_format)
+                new_image = self.glance.images.create(name=image_dict['name'])
+                if image_dict['location'].startswith("http"):
+                    # TODO there is not a method to direct download. It must be downloaded locally with requests
+                    raise vimconn.vimconnNotImplemented("Cannot create image from URL")
                 else: #local path
                     with open(image_dict['location']) as fimage:
-                        new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
-                            container_format="bare", data=fimage, disk_format=disk_format)
-                #insert metadata. We cannot use 'new_image.properties.setdefault' 
-                #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
-                new_image_nova=self.nova.images.find(id=new_image.id)
-                new_image_nova.metadata.setdefault('location',image_dict['location'])
+                        self.glance.images.upload(new_image.id, fimage)
+                        #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
+                        #    container_format="bare", data=fimage, disk_format=disk_format)
                 metadata_to_load = image_dict.get('metadata')
-                if metadata_to_load:
-                    for k,v in yaml.load(metadata_to_load).iteritems():
-                        new_image_nova.metadata.setdefault(k,v)
+                #TODO location is a reserved word for current openstack versions. Use another word
+                metadata_to_load['location'] = image_dict['location']
+                self.glance.images.update(new_image.id, **metadata_to_load)
                 return new_image.id
             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
                 self._format_exception(e)
@@ -813,7 +820,7 @@ class vimconnector(vimconn.vimconnector):
         '''
         try:
             self._reload_connection()
-            self.nova.images.delete(image_id)
+            self.glance.images.delete(image_id)
             return image_id
         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
             self._format_exception(e)
@@ -822,7 +829,7 @@ class vimconnector(vimconn.vimconnector):
         '''Get the image id from image path in the VIM database. Returns the image_id'''
         try:
             self._reload_connection()
-            images = self.nova.images.list()
+            images = self.glance.images.list()
             for image in images:
                 if image.metadata.get("location")==path:
                     return image.id
@@ -845,17 +852,18 @@ class vimconnector(vimconn.vimconnector):
             self._reload_connection()
             filter_dict_os = filter_dict.copy()
             #First we filter by the available filter fields: name, id. The others are removed.
-            filter_dict_os.pop('checksum', None)
-            image_list = self.nova.images.findall(**filter_dict_os)
-            if len(image_list) == 0:
-                return []
-            #Then we filter by the rest of filter fields: checksum
+            image_list = self.glance.images.list()
             filtered_list = []
             for image in image_list:
                 try:
-                    image_class = self.glance.images.get(image.id)
-                    if 'checksum' not in filter_dict or image_class['checksum'] == filter_dict.get('checksum'):
-                        filtered_list.append(image_class.copy())
+                    if filter_dict.get("name") and image["name"] != filter_dict["name"]:
+                        continue
+                    if filter_dict.get("id") and image["id"] != filter_dict["id"]:
+                        continue
+                    if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
+                        continue
+
+                    filtered_list.append(image.copy())
                 except gl1Exceptions.HTTPNotFound:
                     pass
             return filtered_list
@@ -873,8 +881,8 @@ class vimconnector(vimconn.vimconnector):
                 return True
             if vm_status == 'ERROR':
                 return False
-            time.sleep(1)
-            elapsed_time += 1
+            time.sleep(5)
+            elapsed_time += 5
 
         # if we exceeded the timeout rollback
         if elapsed_time >= server_timeout:
@@ -972,6 +980,7 @@ class vimconnector(vimconn.vimconnector):
             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
                 'size': (mandatory) string with the size of the disk in GB
+                'vim_id' (optional) should use this existing volume id
             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
                 availability_zone_index is None
@@ -1064,7 +1073,7 @@ class vimconnector(vimconn.vimconnector):
 
                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
                 # As a workaround we wait until the VM is active and then disable the port-security
-                if net.get("port_security") == False:
+                if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
                     no_secured_ports.append(new_port["port"]["id"])
 
             # if metadata_vpci:
@@ -1086,31 +1095,34 @@ class vimconnector(vimconn.vimconnector):
 
             # Create additional volumes in case these are present in disk_list
             base_disk_index = ord('b')
-            if disk_list != None:
+            if disk_list:
                 block_device_mapping = {}
                 for disk in disk_list:
-                    if 'image_id' in disk:
-                        volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
-                                    chr(base_disk_index), imageRef = disk['image_id'])
+                    if disk.get('vim_id'):
+                        block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
                     else:
-                        volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
-                                    chr(base_disk_index))
-                    created_items["volume:" + str(volume.id)] = True
-                    block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
+                        if 'image_id' in disk:
+                            volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
+                                                                chr(base_disk_index), imageRef=disk['image_id'])
+                        else:
+                            volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
+                                                                chr(base_disk_index))
+                        created_items["volume:" + str(volume.id)] = True
+                        block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
                     base_disk_index += 1
 
-                # Wait until volumes are with status available
-                keep_waiting = True
+                # Wait until created volumes are with status available
                 elapsed_time = 0
-                while keep_waiting and elapsed_time < volume_timeout:
-                    keep_waiting = False
-                    for volume_id in block_device_mapping.itervalues():
-                        if self.cinder.volumes.get(volume_id).status != 'available':
-                            keep_waiting = True
-                    if keep_waiting:
-                        time.sleep(1)
-                        elapsed_time += 1
-
+                while elapsed_time < volume_timeout:
+                    for created_item in created_items:
+                        v, _, volume_id = created_item.partition(":")
+                        if v == 'volume':
+                            if self.cinder.volumes.get(volume_id).status != 'available':
+                                break
+                    else:  # all ready: break from while
+                        break
+                    time.sleep(5)
+                    elapsed_time += 5
                 # If we exceeded the timeout rollback
                 if elapsed_time >= volume_timeout:
                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
@@ -1139,14 +1151,14 @@ class vimconnector(vimconn.vimconnector):
 
             for port_id in no_secured_ports:
                 try:
-                    self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
+                    self.neutron.update_port(port_id,
+                                             {"port": {"port_security_enabled": False, "security_groups": None}})
                 except Exception as e:
-                    self.logger.error("It was not possible to disable port security for port {}".format(port_id))
-                    raise
-
+                    raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
+                        port_id))
             # print "DONE :-)", server
 
-            pool_id = None
+            pool_id = None
             if external_network:
                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
             for floating_network in external_network:
@@ -1162,10 +1174,11 @@ class vimconnector(vimconn.vimconnector):
                                     continue
                             free_floating_ip = ip.get("floating_ip_address")
                         else:
-                            if isinstance(floating_network['floating_ip'], str):
+                            if isinstance(floating_network['floating_ip'], str) and \
+                                floating_network['floating_ip'].lower() != "true":
                                 pool_id = floating_network['floating_ip']
                             else:
-                                #Find the external network
+                                # Find the external network
                                 external_nets = list()
                                 for net in self.neutron.list_networks()['networks']:
                                     if net['router:external']:
@@ -1183,7 +1196,7 @@ class vimconnector(vimconn.vimconnector):
                                 pool_id = external_nets[0].get('id')
                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
                             try:
-                                #self.logger.debug("Creating floating IP")
+                                # self.logger.debug("Creating floating IP")
                                 new_floating_ip = self.neutron.create_floatingip(param)
                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
                             except Exception as e:
@@ -1196,13 +1209,15 @@ class vimconnector(vimconn.vimconnector):
                                 server.add_floating_ip(free_floating_ip, fix_ip)
                                 assigned = True
                             except Exception as e:
+                                # openstack need some time after VM creation to asign an IP. So retry if fails
                                 vm_status = self.nova.servers.get(server.id).status
                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
                                     if time.time() - vm_start_time < server_timeout:
                                         time.sleep(5)
                                         continue
-                                raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e),
-                                                               http_code=vimconn.HTTP_Conflict)
+                                raise vimconn.vimconnException(
+                                    "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
+                                    http_code=vimconn.HTTP_Conflict)
 
                 except Exception as e:
                     if not floating_network['exit_on_floating_ip_error']:
@@ -1391,7 +1406,7 @@ class vimconnector(vimconn.vimconnector):
                 #get interfaces
                 try:
                     self._reload_connection()
-                    port_dict=self.neutron.list_ports(device_id=vm_id)
+                    port_dict = self.neutron.list_ports(device_id=vm_id)
                     for port in port_dict["ports"]:
                         interface={}
                         try:
@@ -1425,16 +1440,20 @@ class vimconnector(vimconn.vimconnector):
                             interface["vlan"] = network['network'].get('provider:segmentation_id')
                         ips=[]
                         #look for floating ip address
-                        floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
-                        if floating_ip_dict.get("floatingips"):
-                            ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
+                        try:
+                            floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
+                            if floating_ip_dict.get("floatingips"):
+                                ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
+                        except Exception:
+                            pass
 
                         for subnet in port["fixed_ips"]:
                             ips.append(subnet["ip_address"])
                         interface["ip_address"] = ";".join(ips)
                         vm["interfaces"].append(interface)
                 except Exception as e:
-                    self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
+                    self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
+                                      exc_info=True)
             except vimconn.vimconnNotFoundException as e:
                 self.logger.error("Exception getting vm status: %s", str(e))
                 vm['status'] = "DELETED"