Merge branch 'v1.1'
[osm/SO.git] / rwcal / plugins / vala / rwcal_openstack / rwcal_openstack.py
index d3c0b97..2be896a 100644 (file)
@@ -20,6 +20,8 @@ import logging
 import os
 import subprocess
 import uuid
+import tempfile
+import yaml
 
 import rift.rwcal.openstack as openstack_drv
 import rw_status
@@ -27,6 +29,10 @@ import rift.cal.rwcal_status as rwcal_status
 import rwlogger
 import neutronclient.common.exceptions as NeutronException
 import keystoneclient.exceptions as KeystoneExceptions
+import tornado
+import gi
+
+gi.require_version('RwSdn', '1.0')
 
 from gi.repository import (
     GObject,
@@ -34,7 +40,7 @@ from gi.repository import (
     RwTypes,
     RwcalYang)
 
-PREPARE_VM_CMD = "prepare_vm.py --auth_url {auth_url} --username {username} --password {password} --tenant_name {tenant_name} --mgmt_network {mgmt_network} --server_id {server_id} --port_metadata"
+PREPARE_VM_CMD = "prepare_vm.py --auth_url {auth_url} --username {username} --password {password} --tenant_name {tenant_name} --region {region} --user_domain {user_domain} --project_domain {project_domain} --mgmt_network {mgmt_network} --server_id {server_id} --port_metadata "
 
 rwstatus_exception_map = { IndexError: RwTypes.RwStatus.NOTFOUND,
                            KeyError: RwTypes.RwStatus.NOTFOUND,
@@ -89,7 +95,8 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                                          mgmt_network        = account.openstack.mgmt_network,
                                          cert_validate       = account.openstack.cert_validate,
                                          user_domain_name    = account.openstack.user_domain,
-                                         project_domain_name = account.openstack.project_domain)
+                                         project_domain_name = account.openstack.project_domain,
+                                         region              = account.openstack.region)
             except (KeystoneExceptions.Unauthorized, KeystoneExceptions.AuthorizationFailure,
                         NeutronException.NotFound) as e:
                 raise
@@ -363,6 +370,7 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
             image = drv.glance_image_get(image_id)
         return RwcalOpenstackPlugin._fill_image_info(image)
 
+    # This is being deprecated. Please do not use for new SW development
     @rwstatus(ret_on_failure=[""])
     def do_create_vm(self, account, vminfo):
         """Create a new virtual machine.
@@ -382,14 +390,12 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
 
         with self._use_driver(account) as drv:
             ### If floating_ip is required and we don't have one, better fail before any further allocation
+            pool_name = None
+            floating_ip = False
             if vminfo.has_field('allocate_public_address') and vminfo.allocate_public_address:
                 if account.openstack.has_field('floating_ip_pool'):
                     pool_name = account.openstack.floating_ip_pool
-                else:
-                    pool_name = None
-                floating_ip = self._allocate_floating_ip(drv, pool_name)
-            else:
-                floating_ip = None
+                floating_ip = True
 
         if vminfo.has_field('cloud_init') and vminfo.cloud_init.has_field('userdata'):
             kwargs['userdata']  = vminfo.cloud_init.userdata
@@ -1181,6 +1187,8 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         if ('fixed_ips' in port_info) and (len(port_info['fixed_ips']) >= 1):
             if 'ip_address' in port_info['fixed_ips'][0]:
                 c_point.ip_address = port_info['fixed_ips'][0]['ip_address']
+        if 'mac_address' in port_info :
+            c_point.mac_addr = port_info['mac_address']
         if port_info['status'] == 'ACTIVE':
             c_point.state = 'active'
         else:
@@ -1229,7 +1237,7 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         return link
 
     @staticmethod
-    def _fill_vdu_info(vm_info, flavor_info, mgmt_network, port_list, server_group, volume_list = None):
+    def _fill_vdu_info(drv, vm_info, flavor_info, mgmt_network, port_list, server_group, volume_list = None):
         """Create a GI object for VDUInfoParams
 
         Converts VM information dictionary object returned by openstack
@@ -1257,9 +1265,17 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                             vdu.public_ip = interface['addr']
 
         # Look for any metadata
-        for key, value in vm_info['metadata'].items():
-            if key == 'node_id':
-                vdu.node_id = value
+#        for key, value in vm_info['metadata'].items():
+#            if key == 'node_id':
+#                vdu.node_id = value
+#            else:
+#                custommetadata = vdu.supplemental_boot_data.custom_meta_data.add()
+#                custommetadata.name = key
+#                custommetadata.value = str(value)
+
+        # Look for config_drive
+        if ('config_drive' in vm_info):
+            vdu.supplemental_boot_data.boot_data_drive = vm_info['config_drive']
         if ('image' in vm_info) and ('id' in vm_info['image']):
             vdu.image_id = vm_info['image']['id']
         if ('flavor' in vm_info) and ('id' in vm_info['flavor']):
@@ -1298,6 +1314,16 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                    continue
                 volr.name = vol_name
                 volr.volume_id = os_volume['volumeId']
+                try:
+                   vol_details = drv.cinder_volume_get(volr.volume_id)
+                except:
+                   continue
+                if vol_details is None:
+                   continue
+                for key, value in vol_details.metadata.items():
+                      volmd = volr.custom_meta_data.add()
+                      volmd.name = key
+                      volmd.value = value
 
         return vdu
 
@@ -1473,6 +1499,10 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         else:
             raise NotImplementedError("Port Type: %s not supported" %(c_point.type_yang))
 
+        # By default port gets created with post_security enaled as True
+        if 'port_security_enabled' in c_point:
+            kwargs['port_security_enabled'] = c_point.port_security_enabled
+
         with self._use_driver(account) as drv:
             if c_point.has_field('security_group'):
                 group = drv.neutron_security_group_by_name(c_point.security_group)
@@ -1889,14 +1919,12 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
 
         with self._use_driver(account) as drv:
             ### If floating_ip is required and we don't have one, better fail before any further allocation
+            floating_ip = False
+            pool_name = None
             if vduinfo.has_field('allocate_public_address') and vduinfo.allocate_public_address:
                 if account.openstack.has_field('floating_ip_pool'):
                     pool_name = account.openstack.floating_ip_pool
-                else:
-                    pool_name = None
-                floating_ip = self._allocate_floating_ip(drv, pool_name)
-            else:
-                floating_ip = None
+                floating_ip = True
 
         if vduinfo.has_field('vdu_init') and vduinfo.vdu_init.has_field('userdata'):
             kwargs['userdata'] = vduinfo.vdu_init.userdata
@@ -1910,12 +1938,33 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         kwargs['network_list'] = network_list
 
         metadata = {}
+        files = {}
+        config_drive = False
         # Add all metadata related fields
         if vduinfo.has_field('node_id'):
             metadata['node_id'] = vduinfo.node_id
         if pci_assignement is not None:
             metadata['pci_assignement'] = pci_assignement
+        if vduinfo.has_field('supplemental_boot_data'):
+            if vduinfo.supplemental_boot_data.has_field('custom_meta_data'):
+                for custom_meta_item in vduinfo.supplemental_boot_data.custom_meta_data:
+                    if custom_meta_item.data_type == "STRING":
+                       metadata[custom_meta_item.name] = custom_meta_item.value
+                    elif custom_meta_item.data_type == "JSON":
+                       metadata[custom_meta_item.name] = tornado.escape.json_decode(custom_meta_item.value)
+                    else:
+                       raise OpenstackCALOperationFailure("Create-vdu operation failed. Unsupported data-type {} for custom-meta-data name {} ".format(custom_meta_item.data_type, custom_meta_item.name))
+            if vduinfo.supplemental_boot_data.has_field('config_file'):
+                for custom_config_file in vduinfo.supplemental_boot_data.config_file:
+                    files[custom_config_file.dest] = custom_config_file.source
+
+            if vduinfo.supplemental_boot_data.has_field('boot_data_drive'):
+                if vduinfo.supplemental_boot_data.boot_data_drive is True:
+                     config_drive = True
+                     
         kwargs['metadata'] = metadata
+        kwargs['files'] = files
+        kwargs['config_drive'] = config_drive
 
         if vduinfo.has_field('availability_zone') and vduinfo.availability_zone.has_field('name'):
             kwargs['availability_zone']  = vduinfo.availability_zone
@@ -1928,13 +1977,14 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
             kwargs['scheduler_hints'] = None
 
         kwargs['block_device_mapping_v2'] = None
+        vol_metadata = False
         if vduinfo.has_field('volumes') :
             kwargs['block_device_mapping_v2'] = []
             with self._use_driver(account) as drv:
             # Only support image->volume
                 for volume in vduinfo.volumes:
                     block_map = dict()
-                    block_map['boot_index'] = volume.boot_params.boot_priority
+                    block_map['boot_index'] = volume.boot_priority
                     if "image" in volume:
                         # Support image->volume
                         # Match retrived image info with volume based image name and checksum
@@ -1954,9 +2004,9 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                     block_map['destination_type'] = "volume"
                     block_map['volume_size'] = volume.size
                     block_map['delete_on_termination'] = True
-                    if volume.guest_params.has_field('device_type') and volume.guest_params.device_type == 'cdrom':
+                    if volume.has_field('device_type') and volume.device_type == 'cdrom':
                         block_map['device_type'] = 'cdrom'
-                    if volume.guest_params.has_field('device_bus') and volume.guest_params.device_bus == 'ide':
+                    if volume.has_field('device_bus') and volume.device_bus == 'ide':
                         block_map['disk_bus'] = 'ide'
                     kwargs['block_device_mapping_v2'].append(block_map)
                 
@@ -1964,7 +2014,7 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         with self._use_driver(account) as drv:
             vm_id = drv.nova_server_create(**kwargs)
             if floating_ip:
-                self.prepare_vdu_on_boot(account, vm_id, floating_ip)
+                self.prepare_vdu_on_boot(account, vm_id, floating_ip, pool_name, vduinfo.volumes)
 
         return vm_id
 
@@ -2015,15 +2065,6 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
         mgmt_network_id = None
         with self._use_driver(account) as drv:
             mgmt_network_id = drv._mgmt_network_id
-            ### If floating_ip is required and we don't have one, better fail before any further allocation
-            if vdu_init.has_field('allocate_public_address') and vdu_init.allocate_public_address:
-                if account.openstack.has_field('floating_ip_pool'):
-                    pool_name = account.openstack.floating_ip_pool
-                else:
-                    pool_name = None
-                floating_ip = self._allocate_floating_ip(drv, pool_name)
-            else:
-                floating_ip = None
 
         port_list = []
         network_list = []
@@ -2105,16 +2146,12 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                           err_str = ("VDU Volume source not supported yet")
                           self.log.error(err_str)
                           raise OpenstackCALOperationFailure("Create-vdu operation failed. Error- %s" % err_str)
-                      if "guest_params" not in volume:
-                          err_str = ("VDU Volume destination parameters '%s' not defined")
+                      if not volume.has_field('device_type'):
+                          err_str = ("VDU Volume destination type not defined")
                           self.log.error(err_str)
                           raise OpenstackCALOperationFailure("Create-vdu operation failed. Error- %s" % err_str)
-                      if not volume.guest_params.has_field('device_type'):
-                          err_str = ("VDU Volume destination type '%s' not defined")
-                          self.log.error(err_str)
-                          raise OpenstackCALOperationFailure("Create-vdu operation failed. Error- %s" % err_str)
-                      if volume.guest_params.device_type not in ['disk', 'cdrom'] :
-                          err_str = ("VDU Volume destination type '%s' not supported" % volume.guest_params.device_type)
+                      if volume.device_type not in ['disk', 'cdrom'] :
+                          err_str = ("VDU Volume destination type '%s' not supported" % volume.device_type)
                           self.log.error(err_str)
                           raise OpenstackCALOperationFailure("Create-vdu operation failed. Error- %s" % err_str)
   
@@ -2133,7 +2170,6 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                 vm.user_tags.pci_assignement = pci_assignement
 
             vm_id = self._create_vm(account, vdu_init, pci_assignement=pci_assignement, server_group=server_group, port_list=port_list, network_list=vm_network_list, imageinfo_list = imageinfo_list)
-            self.prepare_vdu_on_boot(account, vm_id, floating_ip)
             return vm_id
 
     def prepare_vpci_metadata(self, drv, vdu_init):
@@ -2179,16 +2215,38 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
 
 
 
-    def prepare_vdu_on_boot(self, account, server_id, floating_ip):
-        cmd = PREPARE_VM_CMD.format(auth_url     = account.openstack.auth_url,
-                                    username     = account.openstack.key,
-                                    password     = account.openstack.secret,
-                                    tenant_name  = account.openstack.tenant,
-                                    mgmt_network = account.openstack.mgmt_network,
-                                    server_id    = server_id)
-
-        if floating_ip is not None:
-            cmd += (" --floating_ip "+ floating_ip.ip)
+    def prepare_vdu_on_boot(self, account, server_id, floating_ip,  pool_name, volumes=None):
+        cmd = PREPARE_VM_CMD.format(auth_url       = account.openstack.auth_url,
+                                    username       = account.openstack.key,
+                                    password       = account.openstack.secret,
+                                    tenant_name    = account.openstack.tenant,
+                                    region         = account.openstack.region,
+                                    user_domain    = account.openstack.user_domain,
+                                    project_domain = account.openstack.project_domain,
+                                    mgmt_network   = account.openstack.mgmt_network,
+                                    server_id      = server_id)
+        if floating_ip:
+            cmd += " --floating_ip"
+        if pool_name:
+            cmd += (" --pool_name " + pool_name)
+
+        vol_metadata = False
+        if volumes is not None:
+            for volume in volumes:
+                if volume.has_field('custom_meta_data'):
+                    vol_metadata = True
+                    break
+        
+        if vol_metadata is True:       
+            tmp_file = None
+            with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
+                 vol_list = list()
+                 for volume in volumes:
+                    vol_dict = volume.as_dict()
+                    vol_list.append(vol_dict)
+
+                 yaml.dump(vol_list, tmp_file)
+            cmd += (" --vol_metadata {}").format(tmp_file.name)
 
         exec_path = 'python3 ' + os.path.dirname(openstack_drv.__file__)
         exec_cmd = exec_path+'/'+cmd
@@ -2285,7 +2343,7 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
             openstack_group_list = drv.nova_server_group_list()
             server_group = [ i['name'] for i in openstack_group_list if vm['id'] in i['members']]
             openstack_srv_volume_list = drv.nova_volume_list(vm['id'])
-            vdu_info = RwcalOpenstackPlugin._fill_vdu_info(vm,
+            vdu_info = RwcalOpenstackPlugin._fill_vdu_info(drv, vm,
                                                            flavor_info,
                                                            account.openstack.mgmt_network,
                                                            port_list,
@@ -2334,7 +2392,7 @@ class RwcalOpenstackPlugin(GObject.Object, RwCal.Cloud):
                 server_group = [ i['name'] for i in openstack_group_list if vm['id'] in i['members']]
 
                 openstack_srv_volume_list = drv.nova_volume_list(vm['id'])
-                vdu = RwcalOpenstackPlugin._fill_vdu_info(vm,
+                vdu = RwcalOpenstackPlugin._fill_vdu_info(drv, vm,
                                                           flavor_info,
                                                           account.openstack.mgmt_network,
                                                           port_list,