From: Paolo Lungaroni <paolo.lungaroni@cnit.it>
Date: Wed, 14 Feb 2018 17:05:02 +0000 (+0100)
Subject: Bug 462 (Enhancement) - Add support for Xen and Unikernels
X-Git-Tag: v3.1.0
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=50557c2e90ad65d8d9459063bd83d5e0b2e02f37;p=osm%2Fopenvim.git

Bug 462 (Enhancement) - Add support for Xen and Unikernels

Change-Id: I1ea87eccb96d43f52e43a6e4c4232a82c24a8c46
Signed-off-by: Paolo Lungaroni <paolo.lungaroni@cnit.it>
---

diff --git a/database_utils/migrate_vim_db.sh b/database_utils/migrate_vim_db.sh
index e8415e4..9d59b2b 100755
--- a/database_utils/migrate_vim_db.sh
+++ b/database_utils/migrate_vim_db.sh
@@ -33,7 +33,7 @@ DBPORT="3306"
 DBNAME="vim_db"
 QUIET_MODE=""
 #TODO update it with the last database version
-LAST_DB_VERSION=22
+LAST_DB_VERSION=23
 
 # Detect paths
 MYSQL=$(which mysql)
@@ -190,6 +190,7 @@ fi
 #[ $OPENVIM_VER_NUM -ge 5017 ] && DATABASE_TARGET_VER_NUM=20   #0.5.17  => 20
 #[ $OPENVIM_VER_NUM -ge 5018 ] && DATABASE_TARGET_VER_NUM=21   #0.5.18  => 21
 #[ $OPENVIM_VER_NUM -ge 5021 ] && DATABASE_TARGET_VER_NUM=22   #0.5.21  => 22
+#[ $OPENVIM_VER_NUM -ge 5024 ] && DATABASE_TARGET_VER_NUM=23   #0.5.24  => 23
 # TODO ... put next versions here
 
 function upgrade_to_1(){
@@ -770,6 +771,25 @@ function downgrade_from_22(){
     sql "DELETE FROM schema_version WHERE version_int = '22';"
 }
 
+function upgrade_to_23(){
+    echo "    Add 'hypervisor' and 'os_type' column to 'instances' table"
+    sql "ALTER TABLE instances ADD COLUMN hypervisor enum('kvm','xen-unik','xenhvm') NOT NULL DEFAULT 'kvm' AFTER flavor_id;"
+    sql "ALTER TABLE instances ADD COLUMN os_image_type VARCHAR(24) NOT NULL DEFAULT 'other' AFTER hypervisor;"
+    echo "    Add 'hypervisors' column to 'hosts' table"
+    sql "ALTER TABLE hosts ADD COLUMN hypervisors VARCHAR(255) NOT NULL DEFAULT 'kvm' AFTER features;"
+    sql "INSERT INTO schema_version (version_int, version, openvim_ver, comments, date) "\
+        "VALUES (23, '0.23', '0.5.24', 'Add hypervisor, os_type to instances and add hypervisors to hosts', '2018-03-20');"
+}
+
+function downgrade_from_23(){
+    echo "    Remove 'hypervisor' and 'os_type' column from 'instances' table"
+    sql "ALTER TABLE instances DROP COLUMN hypervisor;"
+    sql "ALTER TABLE instances DROP COLUMN os_image_type;"
+    echo "    Remove 'hypervisors' column from 'hosts' table"
+    sql "ALTER TABLE hosts DROP COLUMN hypervisors;"
+    sql "DELETE FROM schema_version WHERE version_int = '23';"
+}
+
 # TODO ... put functions here
 
 # echo "db version = "${DATABASE_VER_NUM}
diff --git a/osm_openvim/host_thread.py b/osm_openvim/host_thread.py
index 78be1c9..34e8f07 100644
--- a/osm_openvim/host_thread.py
+++ b/osm_openvim/host_thread.py
@@ -50,7 +50,7 @@ class host_thread(threading.Thread):
     lvirt_module = None
 
     def __init__(self, name, host, user, db, db_lock, test, image_path, host_id, version, develop_mode,
-                 develop_bridge_iface, password=None, keyfile = None, logger_name=None, debug=None):
+                 develop_bridge_iface, password=None, keyfile = None, logger_name=None, debug=None, hypervisors=None):
         """Init a thread to communicate with compute node or ovs_controller.
         :param host_id: host identity
         :param name: name of the thread
@@ -99,6 +99,14 @@ class host_thread(threading.Thread):
         self.pending_terminate_server =[] #list  with pairs (time,server_uuid) time to send a terminate for a server being destroyed
         self.next_update_server_status = 0 #time when must be check servers status
         
+#######        self.hypervisor = "kvm" #hypervisor flag (default: kvm)
+        if hypervisors:
+            self.hypervisors = hypervisors
+        else:
+            self.hypervisors = "kvm"
+
+        self.xen_hyp = True if "xen" in self.hypervisors else False
+
         self.hostinfo = None 
         
         self.queueLock = threading.Lock()
@@ -107,10 +115,16 @@ class host_thread(threading.Thread):
         self.run_command_session = None
         self.error = None
         self.localhost = True if host == 'localhost' else False
-        self.lvirt_conn_uri = "qemu+ssh://{user}@{host}/system?no_tty=1&no_verify=1".format(
-            user=self.user, host=self.host)
+
+        if self.xen_hyp:
+            self.lvirt_conn_uri = "xen+ssh://{user}@{host}/?no_tty=1&no_verify=1".format(
+                user=self.user, host=self.host)
+        else:
+            self.lvirt_conn_uri = "qemu+ssh://{user}@{host}/system?no_tty=1&no_verify=1".format(
+                user=self.user, host=self.host)
         if keyfile:
             self.lvirt_conn_uri += "&keyfile=" + keyfile
+
         self.remote_ip = None
         self.local_ip = None
 
@@ -488,8 +502,13 @@ class host_thread(threading.Thread):
             bus_ide = True if bus=='ide' else False
             
         self.xml_level = 0
+        hypervisor = server.get('hypervisor', 'kvm')
+        os_type_img = server.get('os_image_type', 'other')
 
-        text = "<domain type='kvm'>"
+        if hypervisor[:3] == 'xen':
+            text = "<domain type='xen'>"
+        else:
+            text = "<domain type='kvm'>"
     #get topology
         topo = server_metadata.get('topology', None)
         if topo == None and 'metadata' in dev_list[0]:
@@ -563,12 +582,26 @@ class host_thread(threading.Thread):
             if dev['type']=='cdrom' :
                 boot_cdrom = True
                 break
-        text += self.tab()+ '<os>' + \
-            self.inc_tab() + "<type arch='x86_64' machine='pc'>hvm</type>"
-        if boot_cdrom:
-            text +=  self.tab() + "<boot dev='cdrom'/>" 
-        text +=  self.tab() + "<boot dev='hd'/>" + \
-            self.dec_tab()+'</os>'
+        if hypervisor == 'xenhvm':
+            text += self.tab()+ '<os>' + \
+                self.inc_tab() + "<type arch='x86_64' machine='xenfv'>hvm</type>"
+            text += self.tab() + "<loader type='rom'>/usr/lib/xen/boot/hvmloader</loader>"
+            if boot_cdrom:
+                text +=  self.tab() + "<boot dev='cdrom'/>" 
+            text +=  self.tab() + "<boot dev='hd'/>" + \
+                self.dec_tab()+'</os>'
+        elif hypervisor == 'xen-unik':
+            text += self.tab()+ '<os>' + \
+                self.inc_tab() + "<type arch='x86_64' machine='xenpv'>xen</type>"
+            text +=  self.tab() + "<kernel>" + str(dev_list[0]['source file']) + "</kernel>" + \
+                self.dec_tab()+'</os>'
+        else:
+            text += self.tab()+ '<os>' + \
+                self.inc_tab() + "<type arch='x86_64' machine='pc'>hvm</type>"
+            if boot_cdrom:
+                text +=  self.tab() + "<boot dev='cdrom'/>" 
+            text +=  self.tab() + "<boot dev='hd'/>" + \
+                self.dec_tab()+'</os>'
     #features
         text += self.tab()+'<features>'+\
             self.inc_tab()+'<acpi/>' +\
@@ -587,14 +620,29 @@ class host_thread(threading.Thread):
             self.tab() + "<on_poweroff>preserve</on_poweroff>" + \
             self.tab() + "<on_reboot>restart</on_reboot>" + \
             self.tab() + "<on_crash>restart</on_crash>"
-        text += self.tab() + "<devices>" + \
-            self.inc_tab() + "<emulator>/usr/libexec/qemu-kvm</emulator>" + \
-            self.tab() + "<serial type='pty'>" +\
-            self.inc_tab() + "<target port='0'/>" + \
-            self.dec_tab() + "</serial>" +\
-            self.tab() + "<console type='pty'>" + \
-            self.inc_tab()+ "<target type='serial' port='0'/>" + \
-            self.dec_tab()+'</console>'
+        if hypervisor == 'xenhvm':
+            text += self.tab() + "<devices>" + \
+                self.inc_tab() + "<emulator>/usr/bin/qemu-system-i386</emulator>" + \
+                self.tab() + "<serial type='pty'>" +\
+                self.inc_tab() + "<target port='0'/>" + \
+                self.dec_tab() + "</serial>" +\
+                self.tab() + "<console type='pty'>" + \
+                self.inc_tab()+ "<target type='serial' port='0'/>" + \
+                self.dec_tab()+'</console>' #In some libvirt version may be: <emulator>/usr/lib64/xen/bin/qemu-dm</emulator> (depends on distro)
+        elif hypervisor == 'xen-unik':
+            text += self.tab() + "<devices>" + \
+                self.tab() + "<console type='pty'>" + \
+                self.inc_tab()+ "<target type='xen' port='0'/>" + \
+                self.dec_tab()+'</console>'
+        else:
+            text += self.tab() + "<devices>" + \
+                self.inc_tab() + "<emulator>/usr/libexec/qemu-kvm</emulator>" + \
+                self.tab() + "<serial type='pty'>" +\
+                self.inc_tab() + "<target port='0'/>" + \
+                self.dec_tab() + "</serial>" +\
+                self.tab() + "<console type='pty'>" + \
+                self.inc_tab()+ "<target type='serial' port='0'/>" + \
+                self.dec_tab()+'</console>'
         if windows_os:
             text += self.tab() + "<controller type='usb' index='0'/>" + \
                 self.tab() + "<controller type='ide' index='0'/>" + \
@@ -605,6 +653,15 @@ class host_thread(threading.Thread):
                 self.dec_tab() + "</video>" + \
                 self.tab() + "<memballoon model='virtio'/>" + \
                 self.tab() + "<input type='tablet' bus='usb'/>" #TODO revisar
+        elif hypervisor == 'xen-unik':
+            pass
+        else:
+            text += self.tab() + "<controller type='ide' index='0'/>" + \
+                self.tab() + "<input type='mouse' bus='ps2'/>" + \
+                self.tab() + "<input type='keyboard' bus='ps2'/>" + \
+                self.tab() + "<video>" + \
+                self.inc_tab() + "<model type='cirrus' vram='9216' heads='1'/>" + \
+                self.dec_tab() + "</video>"
 
 #>             self.tab()+'<alias name=\'hostdev0\'/>\n' +\
 #>             self.dec_tab()+'</hostdev>\n' +\
@@ -621,7 +678,7 @@ class host_thread(threading.Thread):
         vd_index = 'a'
         for dev in dev_list:
             bus_ide_dev = bus_ide
-            if dev['type']=='cdrom' or dev['type']=='disk':
+            if (dev['type']=='cdrom' or dev['type']=='disk') and hypervisor != 'xen-unik':
                 if dev['type']=='cdrom':
                     bus_ide_dev = True
                 text += self.tab() + "<disk type='file' device='"+dev['type']+"'>"
@@ -656,6 +713,8 @@ class host_thread(threading.Thread):
                     dev_text = dev_text.replace('__dev__', vd_index)
                     vd_index = chr(ord(vd_index)+1)
                 text += dev_text
+            elif hypervisor == 'xen-unik':
+                pass
             else:
                 return -1, 'Unknown device type ' + dev['type']
 
@@ -696,6 +755,8 @@ class host_thread(threading.Thread):
                 vlan = content[0]['provider'].replace('OVS:', '')
                 text += self.tab() + "<interface type='bridge'>" + \
                         self.inc_tab() + "<source bridge='ovim-" + str(vlan) + "'/>"
+                if hypervisor == 'xenhvm' or hypervisor == 'xen-unik':
+                    text += self.tab() + "<script path='vif-openvswitch'/>"
             else:
                 return -1, 'Unknown Bridge net provider ' + content[0]['provider']
             if model!=None:
@@ -1771,12 +1832,16 @@ class host_thread(threading.Thread):
                 self.logger.error("launch_server ERROR getting server from DB %d %s", result, server_data)
                 return result, server_data
     
+            self.hypervisor = str(server_data['hypervisor'])
+
         #0: get image metadata
             server_metadata = server.get('metadata', {})
             use_incremental = None
              
             if "use_incremental" in server_metadata:
                 use_incremental = False if server_metadata["use_incremental"] == "no" else True
+            if self.xen_hyp == True:
+                use_incremental = False
 
             server_host_files = self.localinfo['server_files'].get( server['uuid'], {})
             if rebuild:
@@ -2387,6 +2452,7 @@ def create_server(server, db, db_lock, only_of_ports):
     elif requirements['vcpus']==0:
         return (-1, "Processor information not set neither at extended field not at vcpus")    
 
+    if 'hypervisor' in server: requirements['hypervisor'] = server['hypervisor']   #Unikernels extension
 
     db_lock.acquire()
     result, content = db.get_numas(requirements, server.get('host_id', None), only_of_ports)
@@ -2624,6 +2690,8 @@ def create_server(server, db, db_lock, only_of_ports):
     
     if 'description' in server: resources['description'] = server['description']
     if 'name' in server: resources['name'] = server['name']
+    if 'hypervisor' in server: resources['hypervisor'] = server['hypervisor']
+    if 'os_image_type' in server: resources['os_image_type'] = server['os_image_type']
     
     resources['extended'] = {}                          #optional
     resources['extended']['numas'] = []
diff --git a/osm_openvim/httpserver.py b/osm_openvim/httpserver.py
index e3252b9..3de9409 100644
--- a/osm_openvim/httpserver.py
+++ b/osm_openvim/httpserver.py
@@ -153,7 +153,7 @@ http2db_host={'id':'uuid'}
 http2db_tenant={'id':'uuid'}
 http2db_flavor={'id':'uuid','imageRef':'image_id', 'size': 'image_size'}
 http2db_image={'id':'uuid', 'created':'created_at', 'updated':'modified_at', 'public': 'public'}
-http2db_server={'id':'uuid','hostId':'host_id','flavorRef':'flavor_id','imageRef':'image_id','created':'created_at'}
+http2db_server={'id':'uuid','hostId':'host_id','flavorRef':'flavor_id','osImageType':'os_image_type','imageRef':'image_id','created':'created_at'}  #Unikernels extension
 http2db_network={'id':'uuid','provider:vlan':'vlan', 'provider:physical': 'provider'}
 http2db_ofc = {'id': 'uuid'}
 http2db_port={'id':'uuid', 'network_id':'net_id', 'mac_address':'mac', 'device_owner':'type','device_id':'instance_id','binding:switch_port':'switch_port','binding:vlan':'vlan', 'bandwidth':'Mbps'}
@@ -513,7 +513,7 @@ def http_get_hosts():
 
 def get_hosts():
     select_, where_, limit_ = filter_query_string(bottle.request.query, http2db_host,
-                                                  ('id', 'name', 'description', 'status', 'admin_state_up', 'ip_name'))
+                                                  ('id', 'name', 'description', 'status', 'admin_state_up', 'ip_name', 'hypervisors'))  #Unikernels extension
     
     myself = config_dic['http_threads'][ threading.current_thread().name ]
     result, content = myself.db.get_table(FROM='hosts', SELECT=select_, WHERE=where_, LIMIT=limit_)
@@ -656,7 +656,8 @@ def http_post_hosts():
                                     db=config_dic['db'], db_lock=config_dic['db_lock'],
                                     test=host_test_mode, image_path=config_dic['host_image_path'],
                                     version=config_dic['version'], host_id=content['uuid'],
-                                    develop_mode=host_develop_mode, develop_bridge_iface=host_develop_bridge_iface)
+                                    develop_mode=host_develop_mode, develop_bridge_iface=host_develop_bridge_iface,
+                                    hypervisors=host.get('hypervisors', None))  #Unikernels extension
 
             thread.start()
             config_dic['host_threads'][content['uuid']] = thread
diff --git a/osm_openvim/ovim.py b/osm_openvim/ovim.py
index 78c5c5e..8521964 100755
--- a/osm_openvim/ovim.py
+++ b/osm_openvim/ovim.py
@@ -43,9 +43,9 @@ import openflow_conn
 
 __author__ = "Alfonso Tierno, Leonardo Mirabal"
 __date__ = "$06-Feb-2017 12:07:15$"
-__version__ = "0.5.24-r540"
+__version__ = "0.5.24-r542"
 version_date = "Mar 2018"
-database_version = 22      #needed database schema version
+database_version = 23      #needed database schema version
 
 HTTP_Bad_Request =          400
 HTTP_Unauthorized =         401
@@ -199,8 +199,8 @@ class ovim():
         host_develop_bridge_iface = self.config.get('development_bridge', None)
 
         # get host list from data base before starting threads
-        r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid', 'password', 'keyfile'),
-                                     FROM='hosts', WHERE={'status': 'ok'})
+        r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid', 'hypervisors', 'password', 'keyfile'),
+                                     FROM='hosts', WHERE={'status': 'ok'}) #Unikernels extension
         if r < 0:
             raise ovimException("Cannot get hosts from database {}".format(hosts))
 
@@ -215,6 +215,7 @@ class ovim():
                                     version=self.config['version'], host_id=host['uuid'],
                                     develop_mode=host_develop_mode,
                                     develop_bridge_iface=host_develop_bridge_iface,
+                                    hypervisors=host['hypervisors'],  #Unikernels extension
                                     logger_name=self.logger_name + ".host." + host['name'],
                                     debug=self.config.get('log_level_host'))
 
diff --git a/osm_openvim/vim_db.py b/osm_openvim/vim_db.py
index ec29943..abeb0fd 100644
--- a/osm_openvim/vim_db.py
+++ b/osm_openvim/vim_db.py
@@ -248,6 +248,13 @@ class vim_db():
         w = sql_dict.get('WHERE')
         if w:
             where_and = " AND ".join(map( lambda x: str(x) + (" is Null" if w[x] is None else "='"+str(w[x])+"'"),  w.keys()) )
+        w = sql_dict.get('WHERE_LIKE')  #Unikernels extension -START-
+        if w:
+            where_and_like = " AND ".join(map( lambda x: str(x) + (" is Null" if w[x] is None else " LIKE '"+str(w[x])+"'"),  w.keys()) )
+            if where_and:
+                where_and += " AND " + where_and_like
+            else:
+                where_and = where_and_like  #Unikernels extension -END-
         w = sql_dict.get('WHERE_NOT')
         if w:
             where_and_not = " AND ".join(map( lambda x: str(x) + (" is not Null" if w[x] is None else "!='"+str(w[x])+"'"),  w.keys()) )
@@ -474,9 +481,9 @@ class vim_db():
                 with self.con:
                     self.cur = self.con.cursor(mdb.cursors.DictCursor)
                     #get HOST
-                    cmd = "SELECT uuid, user, password, keyfile, name, ip_name, description, ranking, admin_state_up, "\
+                    cmd = "SELECT uuid, user, password, keyfile, name, ip_name, description, hypervisors, ranking, admin_state_up, "\
                           "DATE_FORMAT(created_at,'%Y-%m-%dT%H:%i:%s') as created_at "\
-                          "FROM hosts WHERE " + where_filter
+                          "FROM hosts WHERE " + where_filter  #Unikernels extension
                     self.logger.debug(cmd) 
                     self.cur.execute(cmd)
                     if self.cur.rowcount == 0:
@@ -1054,8 +1061,8 @@ class vim_db():
                 with self.con:
                     self.cur = self.con.cursor(mdb.cursors.DictCursor)
                     #get INSTANCE
-                    cmd = "SELECT uuid, name, description, progress, host_id, flavor_id, image_id, status, last_error, "\
-                        "tenant_id, ram, vcpus, created_at FROM instances WHERE uuid='{}'".format(instance_id)
+                    cmd = "SELECT uuid, name, description, progress, host_id, flavor_id, image_id, status, hypervisor, os_image_type, last_error, "\
+                        "tenant_id, ram, vcpus, created_at FROM instances WHERE uuid='{}'".format(instance_id) #Unikernels extension
                     self.logger.debug(cmd)
                     self.cur.execute(cmd)
                     if self.cur.rowcount == 0 : return 0, "instance '" + str(instance_id) +"'not found."
@@ -1198,6 +1205,19 @@ class vim_db():
                         #self.logger.debug(error_text)
                         return -1, error_text
                     
+                    if not 'hypervisor' in requirements:        #Unikernels extension -END-
+                        requirements['hypervisor'] = "kvm"
+                    for valid_host in valid_hosts:
+                        if not 'hypervisors' in valid_host:
+                            valid_host['hypervisors'] = "kvm"
+
+                    valid_hosts = tuple(valid_host for valid_host in valid_hosts if requirements['hypervisor'] in valid_host['hypervisors'].split(","))
+
+                    if len(valid_hosts)<=0:
+                        error_text = 'No room at data center. Cannot find a host with %s hypervisor or not have enough resources available' % (str(requirements['hypervisor']))
+                        #self.logger.debug(error_text)
+                        return -1, error_text                   #Unikernels extension -END-
+
                     #elif req_numa != None:
                     #Find valid numa nodes for memory requirements
                     self.cur = self.con.cursor(mdb.cursors.DictCursor)
diff --git a/osm_openvim/vim_schema.py b/osm_openvim/vim_schema.py
index b7629f5..87ab3e2 100644
--- a/osm_openvim/vim_schema.py
+++ b/osm_openvim/vim_schema.py
@@ -276,6 +276,7 @@ host_data_schema={
         "features": description_schema,
         "ranking": integer0_schema,
         "autodiscover": {"type": "boolean"},    # try to discover host parameters instead of providing in this schema
+        "hypervisors": description_schema,   #Unikernels extension
         "devices": {
             "type": "array", 
             "items": {
@@ -544,6 +545,8 @@ server_new_schema = {
                 "hostId":id_schema,
                 "flavorRef":id_schema,
                 "imageRef":id_schema,
+                "hypervisor":{"type":"string", "enum":["kvm","xen-unik","xenhvm"]}, #Unikernels extension
+                "osImageType":{"type":"string", "enum":["clickos","other"]}, #Unikernels extension
                 "extended": extended_schema,
                 "networks":networks_schema
             },