Empty disk support added to openvim

[osm/openvim.git] / osm_openvim / ovim.py

diff --git a/osm_openvim/ovim.py b/osm_openvim/ovim.py
index f253912..82d06ce 100755 (executable)
--- a/osm_openvim/ovim.py
+++ b/osm_openvim/ovim.py
@@ -22,21 +22,16 @@
 # contact with: nfvlabs@tid.es
 ##
 
 # contact with: nfvlabs@tid.es
 ##
 

-'''
+"""

 This is the thread for the http server North API. 
 Two thread will be launched, with normal and administrative permissions.
 This is the thread for the http server North API. 
 Two thread will be launched, with normal and administrative permissions.

-'''
-
-__author__ = "Alfonso Tierno, Leonardo Mirabal"
-__date__ = "$06-Feb-2017 12:07:15$"
-__version__ = "0.5.10-r526"
-version_date = "Apr 2017"
-database_version = "0.17"      #expected database schema version
+"""

 
 import threading
 import vim_db
 import logging
 # import imp
 
 import threading
 import vim_db
 import logging
 # import imp

+import os.path

 import argparse
 from netaddr import IPNetwork
 from jsonschema import validate as js_v, exceptions as js_e
 import argparse
 from netaddr import IPNetwork
 from jsonschema import validate as js_v, exceptions as js_e


@@ -45,6 +40,11 @@ import dhcp_thread as dt
 import openflow_thread as oft
 import openflow_conn
 
 import openflow_thread as oft
 import openflow_conn
 

+__author__ = "Alfonso Tierno, Leonardo Mirabal"
+__date__ = "$06-Feb-2017 12:07:15$"
+__version__ = "0.5.17-r533"
+version_date = "Jun 2017"
+database_version = 20      #needed database schema version

 
 HTTP_Bad_Request =          400
 HTTP_Unauthorized =         401
 
 HTTP_Bad_Request =          400
 HTTP_Unauthorized =         401


@@ -100,8 +100,6 @@ class ovim():
         self.logger = logging.getLogger(self.logger_name)
         self.db = None
         self.db = self._create_database_connection()
         self.logger = logging.getLogger(self.logger_name)
         self.db = None
         self.db = self._create_database_connection()

-        self.db_lock = None
-        self.db_of = None

         self.of_test_mode = False
 
     def _create_database_connection(self):
         self.of_test_mode = False
 
     def _create_database_connection(self):


@@ -144,11 +142,11 @@ class ovim():
 
             ips = IPNetwork(cidr)
             if "dhcp_first_ip" not in network:
 
             ips = IPNetwork(cidr)
             if "dhcp_first_ip" not in network:

-                network["dhcp_first_ip"] = str(ips[2])
+                network["dhcp_first_ip"] = str(ips[3])

             if "dhcp_last_ip" not in network:
                 network["dhcp_last_ip"] = str(ips[-2])
             if "gateway_ip" not in network:
             if "dhcp_last_ip" not in network:
                 network["dhcp_last_ip"] = str(ips[-2])
             if "gateway_ip" not in network:

-                network["gateway_ip"] = str(ips[1])
+                network["gateway_ip"] = str(ips[2])

 
             return True
         else:
 
             return True
         else:


@@ -172,19 +170,25 @@ class ovim():
         # if self.running_info:
         #    return  #TODO service can be checked and rebuild broken threads
         r = self.db.get_db_version()
         # if self.running_info:
         #    return  #TODO service can be checked and rebuild broken threads
         r = self.db.get_db_version()

+        db_path = __file__
+        db_path = db_path[:db_path.rfind("/")]
+        if os.path.exists(db_path + "/database_utils/migrate_vim_db.sh"):
+            db_path += "/database_utils"
+        else:
+            db_path += "/../database_utils"
+

         if r[0] < 0:
         if r[0] < 0:

-            raise ovimException("DATABASE is not a VIM one or it is a '0.0' version. Try to upgrade to version '{}' with "\
-                                "'./database_utils/migrate_vim_db.sh'".format(database_version) )
-        elif r[1] != database_version:
-            raise ovimException("DATABASE wrong version '{}'. Try to upgrade/downgrade to version '{}' with "\
-                                "'./database_utils/migrate_vim_db.sh'".format(r[1], database_version) )
+            raise ovimException("DATABASE is not valid. If you think it is corrupted, you can init it with"
+                                " '{db_path}/init_vim_db.sh' script".format(db_path=db_path))
+        elif r[0] != database_version:
+            raise ovimException("DATABASE wrong version '{current}'. Try to upgrade/downgrade to version '{target}'"
+                                " with '{db_path}/migrate_vim_db.sh {target}'".format(
+                                current=r[0], target=database_version,  db_path=db_path))

         self.logger.critical("Starting ovim server version: '{} {}' database version '{}'".format(
             self.get_version(), self.get_version_date(), self.get_database_version()))
         # create database connection for openflow threads
         self.logger.critical("Starting ovim server version: '{} {}' database version '{}'".format(
             self.get_version(), self.get_version_date(), self.get_database_version()))
         # create database connection for openflow threads

-        self.db_of = self._create_database_connection()
-        self.config["db"] = self.db_of
-        self.db_lock = threading.Lock()
-        self.config["db_lock"] = self.db_lock
+        self.config["db"] = self._create_database_connection()
+        self.config["db_lock"] = threading.Lock()

 
         self.of_test_mode = False if self.config['mode'] == 'normal' or self.config['mode'] == "OF only" else True
         # precreate interfaces; [bridge:<host_bridge_name>, VLAN used at Host, uuid of network camping in this bridge,
 
         self.of_test_mode = False if self.config['mode'] == 'normal' or self.config['mode'] == "OF only" else True
         # precreate interfaces; [bridge:<host_bridge_name>, VLAN used at Host, uuid of network camping in this bridge,


@@ -228,7 +232,8 @@ class ovim():
         dhcp_params = self.config.get("dhcp_server")
         if dhcp_params:
             thread = dt.dhcp_thread(dhcp_params=dhcp_params, test=host_test_mode, dhcp_nets=self.config["dhcp_nets"],
         dhcp_params = self.config.get("dhcp_server")
         if dhcp_params:
             thread = dt.dhcp_thread(dhcp_params=dhcp_params, test=host_test_mode, dhcp_nets=self.config["dhcp_nets"],

-                                    db=self.db_of, db_lock=self.db_lock, logger_name=self.logger_name + ".dhcp",
+                                    db=self.config["db"], db_lock=self.config["db_lock"],
+                                    logger_name=self.logger_name + ".dhcp",

                                     debug=self.config.get('log_level_of'))
             thread.start()
             self.config['dhcp_thread'] = thread
                                     debug=self.config.get('log_level_of'))
             thread.start()
             self.config['dhcp_thread'] = thread


@@ -239,16 +244,20 @@ class ovim():
         host_develop_bridge_iface = self.config.get('development_bridge', None)
 
         # get host list from data base before starting threads
         host_develop_bridge_iface = self.config.get('development_bridge', None)
 
         # get host list from data base before starting threads

-        r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid'), FROM='hosts', WHERE={'status': 'ok'})
+        r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid', 'password', 'keyfile'),
+                                     FROM='hosts', WHERE={'status': 'ok'})

         if r < 0:
             raise ovimException("Cannot get hosts from database {}".format(hosts))
 
         self.config['host_threads'] = {}
         for host in hosts:
         if r < 0:
             raise ovimException("Cannot get hosts from database {}".format(hosts))
 
         self.config['host_threads'] = {}
         for host in hosts:

-            host['image_path'] = '/opt/VNF/images/openvim'
-            thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=self.db_of,
-                                    db_lock=self.db_lock, test=host_test_mode, image_path=self.config['image_path'],
-                                    version=self.config['version'], host_id=host['uuid'], develop_mode=host_develop_mode,
+            thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=self.config["db"],
+                                    password=host['password'],
+                                    keyfile=host.get('keyfile', self.config["host_ssh_keyfile"]),
+                                    db_lock=self.config["db_lock"], test=host_test_mode,
+                                    image_path=self.config['host_image_path'],
+                                    version=self.config['version'], host_id=host['uuid'],
+                                    develop_mode=host_develop_mode,

                                     develop_bridge_iface=host_develop_bridge_iface,
                                     logger_name=self.logger_name + ".host." + host['name'],
                                     debug=self.config.get('log_level_host'))
                                     develop_bridge_iface=host_develop_bridge_iface,
                                     logger_name=self.logger_name + ".host." + host['name'],
                                     debug=self.config.get('log_level_host'))


@@ -263,13 +272,20 @@ class ovim():
 
         for net in content:
             net_type = net['type']
 
         for net in content:
             net_type = net['type']

-            if (net_type == 'bridge_data' or net_type == 'bridge_man') \
-                    and net["provider"][:4] == 'OVS:' and net["enable_dhcp"] == "true":
+            if (net_type == 'bridge_data' or net_type == 'bridge_man') and \
+                    net["provider"][:4] == 'OVS:' and net["enable_dhcp"] == "true":
+                try:

                     self.launch_dhcp_server(net['vlan'],
                                             net['dhcp_first_ip'],
                                             net['dhcp_last_ip'],
                                             net['cidr'],
                                             net['gateway_ip'])
                     self.launch_dhcp_server(net['vlan'],
                                             net['dhcp_first_ip'],
                                             net['dhcp_last_ip'],
                                             net['cidr'],
                                             net['gateway_ip'])

+                except Exception as e:
+                    self.logger.error("Fail at launching dhcp server for net_id='%s' net_name='%s': %s",
+                                      net["uuid"], net["name"], str(e))
+                    self.db.update_rows("nets", {"status": "ERROR",
+                                                 "last_error": "Fail at launching dhcp server: " + str(e)},
+                                        {"uuid": net["uuid"]})

 
     def _start_of_db_tasks(self):
         """
 
     def _start_of_db_tasks(self):
         """


@@ -371,19 +387,12 @@ class ovim():
                 module = temp_dict['of_controller']
 
             if module not in ovim.of_module:
                 module = temp_dict['of_controller']
 
             if module not in ovim.of_module:

-                for base in ("", "osm_openvim.", "lib_osm_openvim."):
-                    try:
-                        pkg = __import__(base + module)
-                        if base:
-                            of_conn_module = getattr(pkg, module)
-                        else:
-                            of_conn_module = pkg
-                        ovim.of_module[module] = of_conn_module
-                        self.logger.debug("Module load from {}".format(base + module))
-                        break
-                    except Exception as e:
-                        self.logger.warning("Module {} not found {}".format(base + module, e))
-                else:
+                try:
+                    pkg = __import__("osm_openvim." + module)
+                    of_conn_module = getattr(pkg, module)
+                    ovim.of_module[module] = of_conn_module
+                    self.logger.debug("Module load from {}".format("osm_openvim." + module))
+                except Exception as e:

                     self.logger.error("Cannot open openflow controller module of type '%s'", module)
                     raise ovimException("Cannot open openflow controller of type module '{}'"
                                         "Revise it is installed".format(module),
                     self.logger.error("Cannot open openflow controller module of type '%s'", module)
                     raise ovimException("Cannot open openflow controller of type module '{}'"
                                         "Revise it is installed".format(module),


@@ -409,8 +418,11 @@ class ovim():
         #    ofc_net_same_vlan = False
         ofc_net_same_vlan = False
 
         #    ofc_net_same_vlan = False
         ofc_net_same_vlan = False
 

-        thread = oft.openflow_thread(ofc_uuid, of_conn, of_test=self.of_test_mode, db=self.db_of, db_lock=self.db_lock,
-                                     pmp_with_same_vlan=ofc_net_same_vlan, debug=self.config['log_level_of'])
+        thread = oft.openflow_thread(ofc_uuid, of_conn, of_test=self.of_test_mode, db=self.config["db"],
+                                     db_lock=self.config["db_lock"],
+                                     pmp_with_same_vlan=ofc_net_same_vlan,
+                                     logger_name=self.logger_name + ".ofc." + ofc_uuid,
+                                     debug=self.config.get('log_level_of'))

         #r, c = thread.OF_connector.obtain_port_correspondence()
         #if r < 0:
         #    raise ovimException("Cannot get openflow information %s", c)
         #r, c = thread.OF_connector.obtain_port_correspondence()
         #if r < 0:
         #    raise ovimException("Cannot get openflow information %s", c)


@@ -429,9 +441,13 @@ class ovim():
         if 'dhcp_thread' in self.config:
             threads['dhcp'] = (self.config['dhcp_thread'])
 
         if 'dhcp_thread' in self.config:
             threads['dhcp'] = (self.config['dhcp_thread'])
 

-        for thread in threads.values():
+        for thread_id, thread in threads.items():
+            if thread_id == 'openvim_controller':
+                continue

             thread.insert_task("exit")
             thread.insert_task("exit")

-        for thread in threads.values():
+        for thread_id, thread in threads.items():
+            if thread_id == 'openvim_controller':
+                continue

             thread.join()
 
     def get_networks(self, columns=None, db_filter={}, limit=None):
             thread.join()
 
     def get_networks(self, columns=None, db_filter={}, limit=None):


@@ -499,6 +515,7 @@ class ovim():
         net_vlan = network.get("vlan")
         net_bind_net = network.get("bind_net")
         net_bind_type = network.get("bind_type")
         net_vlan = network.get("vlan")
         net_bind_net = network.get("bind_net")
         net_bind_type = network.get("bind_type")

+        net_region = network.get("region")

         name = network["name"]
 
         # check if network name ends with :<vlan_tag> and network exist in order to make and automated bindning
         name = network["name"]
 
         # check if network name ends with :<vlan_tag> and network exist in order to make and automated bindning


@@ -560,7 +577,7 @@ class ovim():
                 bridge_net_name = net_provider[7:]
                 for brnet in self.config['bridge_nets']:
                     if brnet[0] == bridge_net_name:  # free
                 bridge_net_name = net_provider[7:]
                 for brnet in self.config['bridge_nets']:
                     if brnet[0] == bridge_net_name:  # free

-                        if not brnet[3]:
+                        if brnet[3]:

                             raise ovimException("invalid 'provider:physical', "
                                                 "bridge '%s' is already used" % bridge_net_name, HTTP_Conflict)
                         bridge_net = brnet
                             raise ovimException("invalid 'provider:physical', "
                                                 "bridge '%s' is already used" % bridge_net_name, HTTP_Conflict)
                         bridge_net = brnet


@@ -594,8 +611,13 @@ class ovim():
                 net_vlan = bridge_net[1]
         elif net_type == 'bridge_data' or net_type == 'bridge_man' and self.config['network_type'] == 'ovs':
             net_provider = 'OVS'
                 net_vlan = bridge_net[1]
         elif net_type == 'bridge_data' or net_type == 'bridge_man' and self.config['network_type'] == 'ovs':
             net_provider = 'OVS'

+        if not net_region:
+            if net_type == "data" or net_type == "ptp":
+                net_region = "__DATA__"
+            elif net_provider == "OVS":
+                net_region = "__OVS__"

         if not net_vlan and (net_type == "data" or net_type == "ptp" or net_provider == "OVS"):
         if not net_vlan and (net_type == "data" or net_type == "ptp" or net_provider == "OVS"):

-            net_vlan = self.db.get_free_net_vlan()
+            net_vlan = self.db.get_free_net_vlan(net_region)

             if net_vlan < 0:
                 raise ovimException("Error getting an available vlan", HTTP_Internal_Server_Error)
         if net_provider == 'OVS':
             if net_vlan < 0:
                 raise ovimException("Error getting an available vlan", HTTP_Internal_Server_Error)
         if net_provider == 'OVS':


@@ -604,6 +626,7 @@ class ovim():
         network['provider'] = net_provider
         network['type'] = net_type
         network['vlan'] = net_vlan
         network['provider'] = net_provider
         network['type'] = net_type
         network['vlan'] = net_vlan

+        network['region'] = net_region

         dhcp_integrity = True
         if 'enable_dhcp' in network and network['enable_dhcp']:
             dhcp_integrity = self._check_dhcp_data_integrity(network)
         dhcp_integrity = True
         if 'enable_dhcp' in network and network['enable_dhcp']:
             dhcp_integrity = self._check_dhcp_data_integrity(network)


@@ -617,7 +640,7 @@ class ovim():
                 if network["name"] in self.config["dhcp_server"].get("nets", ()):
                     self.config["dhcp_nets"].append(content)
                     self.logger.debug("dhcp_server: add new net", content)
                 if network["name"] in self.config["dhcp_server"].get("nets", ()):
                     self.config["dhcp_nets"].append(content)
                     self.logger.debug("dhcp_server: add new net", content)

-                elif not bridge_net and bridge_net[0] in self.config["dhcp_server"].get("bridge_ifaces", ()):
+                elif bridge_net and bridge_net[0] in self.config["dhcp_server"].get("bridge_ifaces", ()):

                     self.config["dhcp_nets"].append(content)
                     self.logger.debug("dhcp_server: add new net", content, content)
             return content
                     self.config["dhcp_nets"].append(content)
                     self.logger.debug("dhcp_server: add new net", content, content)
             return content


@@ -737,7 +760,7 @@ class ovim():
                 self.config["dhcp_nets"].remove(network_id)
             return content
         else:
                 self.config["dhcp_nets"].remove(network_id)
             return content
         else:

-            raise ovimException("Error deleting  network %s" % network_id, HTTP_Internal_Server_Error)
+            raise ovimException("Error deleting network '{}': {}".format(network_id, content), -result)

 
     def get_openflow_rules(self, network_id=None):
         """
 
     def get_openflow_rules(self, network_id=None):
         """


@@ -826,7 +849,7 @@ class ovim():
             else:
                 raise ovimException("Default Openflow controller not not running", HTTP_Not_Found)
 
             else:
                 raise ovimException("Default Openflow controller not not running", HTTP_Not_Found)
 

-        if ofc_id in self.config['ofcs_thread']:
+        elif ofc_id in self.config['ofcs_thread']:

             conn = self.config['ofcs_thread'][ofc_id].OF_connector
         else:
             raise ovimException("Openflow controller not found with ofc_id={}".format(ofc_id), HTTP_Not_Found)
             conn = self.config['ofcs_thread'][ofc_id].OF_connector
         else:
             raise ovimException("Openflow controller not found with ofc_id={}".format(ofc_id), HTTP_Not_Found)


@@ -1336,19 +1359,21 @@ class ovim():
 
         bridge_ifaces = []
         controller_ip = self.config['ovs_controller_ip']
 
         bridge_ifaces = []
         controller_ip = self.config['ovs_controller_ip']

-        ovs_controller_user = self.config['ovs_controller_user']
+        ovs_controller_user = self.config.get('ovs_controller_user')

 
         host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False
         host_develop_mode = True if self.config['mode'] == 'development' else False
 
         dhcp_host = ht.host_thread(name='openvim_controller', user=ovs_controller_user, host=controller_ip,
 
         host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False
         host_develop_mode = True if self.config['mode'] == 'development' else False
 
         dhcp_host = ht.host_thread(name='openvim_controller', user=ovs_controller_user, host=controller_ip,

-                                   db=self.db_of,
-                                   db_lock=self.db_lock, test=host_test_mode,
-                                   image_path=self.config['image_path'], version=self.config['version'],
+                                   password=self.config.get('ovs_controller_password'),
+                                   keyfile=self.config.get('ovs_controller_keyfile'),
+                                   db=self.config["db"], db_lock=self.config["db_lock"], test=host_test_mode,
+                                   image_path=self.config['host_image_path'], version=self.config['version'],

                                    host_id='openvim_controller', develop_mode=host_develop_mode,
                                    host_id='openvim_controller', develop_mode=host_develop_mode,

-                                   develop_bridge_iface=bridge_ifaces, logger_name=self.logger_name + ".host.controller",
+                                   develop_bridge_iface=bridge_ifaces,
+                                   logger_name=self.logger_name + ".host.controller",

                                    debug=self.config.get('log_level_host'))
                                    debug=self.config.get('log_level_host'))

-
+        # dhcp_host.start()

         self.config['host_threads']['openvim_controller'] = dhcp_host
         if not host_test_mode:
             dhcp_host.ssh_connect()
         self.config['host_threads']['openvim_controller'] = dhcp_host
         if not host_test_mode:
             dhcp_host.ssh_connect()


@@ -1372,7 +1397,7 @@ class ovim():
 
         controller_host = self.get_dhcp_controller()
         controller_host.create_linux_bridge(vlan)
 
         controller_host = self.get_dhcp_controller()
         controller_host.create_linux_bridge(vlan)

-        controller_host.create_dhcp_interfaces(vlan, first_ip, dhcp_netmask)
+        controller_host.create_dhcp_interfaces(vlan, gateway, dhcp_netmask)

         controller_host.launch_dhcp_server(vlan, ip_range, dhcp_netmask, dhcp_path, gateway)
 
 if __name__ == "__main__":
         controller_host.launch_dhcp_server(vlan, ip_range, dhcp_netmask, dhcp_path, gateway)
 
 if __name__ == "__main__":