ovim.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openvim
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 This is the thread for the http server North API.
  26 Two thread will be launched, with normal and administrative permissions.
  27 '''
  28
  29 __author__ = "Alfonso Tierno, Leonardo Mirabal"
  30 __date__ = "$06-Feb-2017 12:07:15$"
  31
  32 import threading
  33 import vim_db
  34 import logging
  35 import threading
  36 import imp
  37 import host_thread as ht
  38 import dhcp_thread as dt
  39 import openflow_thread as oft
  40 from netaddr import IPNetwork, IPAddress, all_matching_cidrs
  41
  42 HTTP_Bad_Request =          400
  43 HTTP_Unauthorized =         401
  44 HTTP_Not_Found =            404
  45 HTTP_Forbidden =            403
  46 HTTP_Method_Not_Allowed =   405
  47 HTTP_Not_Acceptable =       406
  48 HTTP_Request_Timeout =      408
  49 HTTP_Conflict =             409
  50 HTTP_Service_Unavailable =  503
  51 HTTP_Internal_Server_Error= 500
  52
  53
  54 def convert_boolean(data, items):
  55     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
  56     It assumes that bandwidth is well formed
  57     Attributes:
  58         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
  59         'items': tuple of keys to convert
  60     Return:
  61         None
  62     '''
  63     if type(data) is dict:
  64         for k in data.keys():
  65             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
  66                 convert_boolean(data[k], items)
  67             if k in items:
  68                 if type(data[k]) is str:
  69                     if data[k] == "false":
  70                         data[k] = False
  71                     elif data[k] == "true":
  72                         data[k] = True
  73     if type(data) is tuple or type(data) is list:
  74         for k in data:
  75             if type(k) is dict or type(k) is tuple or type(k) is list:
  76                 convert_boolean(k, items)
  77
  78
  79
  80 class ovimException(Exception):
  81     def __init__(self, message, http_code=HTTP_Bad_Request):
  82         self.http_code = http_code
  83         Exception.__init__(self, message)
  84
  85
  86 class ovim():
  87     running_info = {} #TODO OVIM move the info of running threads from config_dic to this static variable
  88     def __init__(self, configuration):
  89         self.config = configuration
  90         self.logger = logging.getLogger(configuration["logger_name"])
  91         self.db = None
  92         self.db =   self._create_database_connection()
  93
  94
  95     def _create_database_connection(self):
  96         db = vim_db.vim_db((self.config["network_vlan_range_start"], self.config["network_vlan_range_end"]),
  97                            self.config['log_level_db']);
  98         if db.connect(self.config['db_host'], self.config['db_user'], self.config['db_passwd'],
  99                       self.config['db_name']) == -1:
 100             # self.logger.error("Cannot connect to database %s at %s@%s", self.config['db_name'], self.config['db_user'],
 101             #              self.config['db_host'])
 102             raise ovimException("Cannot connect to database {} at {}@{}".format(self.config['db_name'],
 103                                                                                 self.config['db_user'],
 104                                                                                 self.config['db_host']) )
 105         return db
 106
 107
 108     def start_service(self):
 109         #if self.running_info:
 110         #    return  #TODO service can be checked and rebuild broken threads
 111         r = self.db.get_db_version()
 112         if r[0]<0:
 113             raise ovimException("DATABASE is not a VIM one or it is a '0.0' version. Try to upgrade to version '{}' with "\
 114                                 "'./database_utils/migrate_vim_db.sh'".format(self.config["database_version"]) )
 115         elif r[1]!=self.config["database_version"]:
 116             raise ovimException("DATABASE wrong version '{}'. Try to upgrade/downgrade to version '{}' with "\
 117                                 "'./database_utils/migrate_vim_db.sh'".format(r[1], self.config["database_version"]) )
 118
 119         # create database connection for openflow threads
 120         db_of = self._create_database_connection()
 121         self.config["db"] = db_of
 122         db_lock = threading.Lock()
 123         self.config["db_lock"] = db_lock
 124
 125         # precreate interfaces; [bridge:<host_bridge_name>, VLAN used at Host, uuid of network camping in this bridge, speed in Gbit/s
 126         self.config['dhcp_nets'] = []
 127         self.config['bridge_nets'] = []
 128         for bridge, vlan_speed in self.config["bridge_ifaces"].items():
 129         # skip 'development_bridge'
 130             if self.config['mode'] == 'development' and self.config['development_bridge'] == bridge:
 131                 continue
 132             self.config['bridge_nets'].append([bridge, vlan_speed[0], vlan_speed[1], None])
 133
 134         # check if this bridge is already used (present at database) for a network)
 135         used_bridge_nets = []
 136         for brnet in self.config['bridge_nets']:
 137             r, nets = db_of.get_table(SELECT=('uuid',), FROM='nets', WHERE={'provider': "bridge:" + brnet[0]})
 138             if r > 0:
 139                 brnet[3] = nets[0]['uuid']
 140                 used_bridge_nets.append(brnet[0])
 141                 if self.config.get("dhcp_server"):
 142                     if brnet[0] in self.config["dhcp_server"]["bridge_ifaces"]:
 143                         self.config['dhcp_nets'].append(nets[0]['uuid'])
 144         if len(used_bridge_nets) > 0:
 145             self.logger.info("found used bridge nets: " + ",".join(used_bridge_nets))
 146         # get nets used by dhcp
 147         if self.config.get("dhcp_server"):
 148             for net in self.config["dhcp_server"].get("nets", ()):
 149                 r, nets = db_of.get_table(SELECT=('uuid',), FROM='nets', WHERE={'name': net})
 150                 if r > 0:
 151                     self.config['dhcp_nets'].append(nets[0]['uuid'])
 152
 153         # get host list from data base before starting threads
 154         r, hosts = db_of.get_table(SELECT=('name', 'ip_name', 'user', 'uuid'), FROM='hosts', WHERE={'status': 'ok'})
 155         if r < 0:
 156             raise ovimException("Cannot get hosts from database {}".format(hosts))
 157         # create connector to the openflow controller
 158         of_test_mode = False if self.config['mode'] == 'normal' or self.config['mode'] == "OF only" else True
 159
 160         if of_test_mode:
 161             OF_conn = oft.of_test_connector({"of_debug": self.config['log_level_of']})
 162         else:
 163             # load other parameters starting by of_ from config dict in a temporal dict
 164             temp_dict = {"of_ip": self.config['of_controller_ip'],
 165                          "of_port": self.config['of_controller_port'],
 166                          "of_dpid": self.config['of_controller_dpid'],
 167                          "of_debug": self.config['log_level_of']
 168                          }
 169             for k, v in self.config.iteritems():
 170                 if type(k) is str and k[0:3] == "of_" and k[0:13] != "of_controller":
 171                     temp_dict[k] = v
 172             if self.config['of_controller'] == 'opendaylight':
 173                 module = "ODL"
 174             elif "of_controller_module" in self.config:
 175                 module = self.config["of_controller_module"]
 176             else:
 177                 module = self.config['of_controller']
 178             module_info = None
 179             try:
 180                 module_info = imp.find_module(module)
 181
 182                 OF_conn = imp.load_module("OF_conn", *module_info)
 183                 try:
 184                     OF_conn = OF_conn.OF_conn(temp_dict)
 185                 except Exception as e:
 186                     self.logger.error("Cannot open the Openflow controller '%s': %s", type(e).__name__, str(e))
 187                     if module_info and module_info[0]:
 188                         file.close(module_info[0])
 189                     exit(-1)
 190             except (IOError, ImportError) as e:
 191                 if module_info and module_info[0]:
 192                     file.close(module_info[0])
 193                 self.logger.error(
 194                     "Cannot open openflow controller module '%s'; %s: %s; revise 'of_controller' field of configuration file.",
 195                     module, type(e).__name__, str(e))
 196                 raise ovimException("Cannot open openflow controller module '{}'; {}: {}; revise 'of_controller' field of configuration file.".fromat(
 197                         module, type(e).__name__, str(e)))
 198
 199
 200                 # create openflow thread
 201         thread = oft.openflow_thread(OF_conn, of_test=of_test_mode, db=db_of, db_lock=db_lock,
 202                                      pmp_with_same_vlan=self.config['of_controller_nets_with_same_vlan'],
 203                                      debug=self.config['log_level_of'])
 204         r, c = thread.OF_connector.obtain_port_correspondence()
 205         if r < 0:
 206             raise ovimException("Cannot get openflow information %s", c)
 207         thread.start()
 208         self.config['of_thread'] = thread
 209
 210         # create dhcp_server thread
 211         host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False
 212         dhcp_params = self.config.get("dhcp_server")
 213         if dhcp_params:
 214             thread = dt.dhcp_thread(dhcp_params=dhcp_params, test=host_test_mode, dhcp_nets=self.config["dhcp_nets"],
 215                                     db=db_of, db_lock=db_lock, debug=self.config['log_level_of'])
 216             thread.start()
 217             self.config['dhcp_thread'] = thread
 218
 219         # Create one thread for each host
 220         host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False
 221         host_develop_mode = True if self.config['mode'] == 'development' else False
 222         host_develop_bridge_iface = self.config.get('development_bridge', None)
 223         self.config['host_threads'] = {}
 224         for host in hosts:
 225             host['image_path'] = '/opt/VNF/images/openvim'
 226             thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=db_of, db_lock=db_lock,
 227                                     test=host_test_mode, image_path=self.config['image_path'], version=self.config['version'],
 228                                     host_id=host['uuid'], develop_mode=host_develop_mode,
 229                                     develop_bridge_iface=host_develop_bridge_iface)
 230             thread.start()
 231             self.config['host_threads'][host['uuid']] = thread
 232
 233         # create ovs dhcp thread
 234         result, content = self.db.get_table(FROM='nets')
 235         if result < 0:
 236             self.logger.error("http_get_ports Error %d %s", result, content)
 237             raise ovimException(str(content), -result)
 238
 239         for net in content:
 240             net_type = net['type']
 241             if net_type == 'bridge_data' or net_type == 'bridge_man' and net["provider"][:4]=="OVS:" and net["enable_dhcp"] == "true":
 242                 if net['enable_dhcp'] == 'true':
 243                     self.launch_dhcp_server(net['vlan'], net['dhcp_first_ip'], net['dhcp_last_ip'], net['cidr'])
 244
 245     def stop_service(self):
 246         threads = self.config.get('host_threads', {})
 247         if 'of_thread' in self.config:
 248             threads['of'] = (self.config['of_thread'])
 249         if 'dhcp_thread' in self.config:
 250             threads['dhcp'] = (self.config['dhcp_thread'])
 251
 252         for thread in threads.values():
 253             thread.insert_task("exit")
 254         for thread in threads.values():
 255             thread.join()
 256
 257
 258     def get_ports(self, columns=None, filter={}, limit=None):
 259         # result, content = my.db.get_ports(where_)
 260         result, content = self.db.get_table(SELECT=columns, WHERE=filter, FROM='ports', LIMIT=limit)
 261         if result < 0:
 262             self.logger.error("http_get_ports Error %d %s", result, content)
 263             raise ovimException(str(content), -result)
 264         else:
 265             convert_boolean(content, ('admin_state_up',))
 266             return content
 267
 268
 269     def new_port(self, port_data):
 270         port_data['type'] = 'external'
 271         if port_data.get('net_id'):
 272             # check that new net has the correct type
 273             result, new_net = self.db.check_target_net(port_data['net_id'], None, 'external')
 274             if result < 0:
 275                 raise ovimException(str(new_net), -result)
 276         # insert in data base
 277         result, uuid = self.db.new_row('ports', port_data, True, True)
 278         if result > 0:
 279             if 'net_id' in port_data:
 280                 r, c = self.config['of_thread'].insert_task("update-net", port_data['net_id'])
 281                 if r < 0:
 282                     self.logger.error("Cannot insert a task for updating network '$s' %s", port_data['net_id'], c)
 283                     #TODO put network in error status
 284             return uuid
 285         else:
 286             raise ovimException(str(uuid), -result)
 287
 288     def delete_port(self, port_id):
 289         # Look for the previous port data
 290         result, ports = self.db.get_table(WHERE={'uuid': port_id, "type": "external"}, FROM='ports')
 291         if result < 0:
 292             raise ovimException("Cannot get port info from database: {}".format(ports), http_code=-result)
 293         # delete from the data base
 294         result, content = self.db.delete_row('ports', port_id)
 295         if result == 0:
 296             raise ovimException("External port '{}' not found".format(port_id), http_code=HTTP_Not_Found)
 297         elif result < 0:
 298             raise ovimException("Cannot delete port from database: {}".format(content), http_code=-result)
 299         # update network
 300         network = ports[0].get('net_id', None)
 301         if network:
 302             # change of net.
 303             r, c = self.config['of_thread'].insert_task("update-net", network)
 304             if r < 0:
 305                 self.logger.error("Cannot insert a task for updating network '$s' %s", network, c)
 306         return content
 307
 308
 309     def edit_port(self, port_id, port_data, admin=True):
 310         # Look for the previous port data
 311         result, content = self.db.get_table(FROM="ports", WHERE={'uuid': port_id})
 312         if result < 0:
 313             raise ovimException("Cannot get port info from database: {}".format(content), http_code=-result)
 314         elif result == 0:
 315             raise ovimException("Port '{}' not found".format(port_id), http_code=HTTP_Not_Found)
 316         port = content[0]
 317         nets = []
 318         host_id = None
 319         result = 1
 320         if 'net_id' in port_data:
 321             # change of net.
 322             old_net = port.get('net_id', None)
 323             new_net = port_data['net_id']
 324             if old_net != new_net:
 325
 326                 if new_net:
 327                     nets.append(new_net)  # put first the new net, so that new openflow rules are created before removing the old ones
 328                 if old_net:
 329                     nets.append(old_net)
 330                 if port['type'] == 'instance:bridge' or port['type'] == 'instance:ovs':
 331                     raise ovimException("bridge interfaces cannot be attached to a different net", http_code=HTTP_Forbidden)
 332                 elif port['type'] == 'external' and not admin:
 333                     raise ovimException("Needed admin privileges",http_code=HTTP_Unauthorized)
 334                 if new_net:
 335                     # check that new net has the correct type
 336                     result, new_net_dict = self.db.check_target_net(new_net, None, port['type'])
 337                     if result < 0:
 338                         raise ovimException("Error {}".format(new_net_dict), http_code=HTTP_Conflict)
 339                 # change VLAN for SR-IOV ports
 340                 if result >= 0 and port["type"] == "instance:data" and port["model"] == "VF":  # TODO consider also VFnotShared
 341                     if new_net:
 342                         port_data["vlan"] = None
 343                     else:
 344                         port_data["vlan"] = new_net_dict["vlan"]
 345                     # get host where this VM is allocated
 346                     result, content = self.db.get_table(FROM="instances", WHERE={"uuid": port["instance_id"]})
 347                     if result > 0:
 348                         host_id = content[0]["host_id"]
 349
 350         # insert in data base
 351         if result >= 0:
 352             result, content = self.db.update_rows('ports', port_data, WHERE={'uuid': port_id}, log=False)
 353
 354         # Insert task to complete actions
 355         if result > 0:
 356             for net_id in nets:
 357                 r, v = self.config['of_thread'].insert_task("update-net", net_id)
 358                 if r < 0:
 359                     self.logger.error("Error updating network '{}' {}".format(r,v))
 360                     # TODO Do something if fails
 361             if host_id:
 362                 r, v = self.config['host_threads'][host_id].insert_task("edit-iface", port_id, old_net, new_net)
 363                 if r < 0:
 364                     self.logger.error("Error updating network '{}' {}".format(r,v))
 365                     # TODO Do something if fails
 366         if result >= 0:
 367             return port_id
 368         else:
 369             raise ovimException("Error {}".format(content), http_code=-result)
 370
 371     def get_dhcp_controller(self):
 372         """
 373         Create an host_thread object for manage openvim controller and not create a thread for itself
 374         :return: dhcp_host openvim controller object
 375         """
 376
 377         if 'openvim_controller' in self.config['host_threads']:
 378             return self.config['host_threads']['openvim_controller']
 379
 380         bridge_ifaces = []
 381         controller_ip = self.config['ovs_controller_ip']
 382         ovs_controller_user = self.config['ovs_controller_user']
 383
 384         host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False
 385         host_develop_mode = True if self.config['mode'] == 'development' else False
 386
 387         dhcp_host = ht.host_thread(name='openvim_controller', user=ovs_controller_user, host=controller_ip,
 388                                    db=self.config['db'],
 389                                    db_lock=self.config['db_lock'], test=host_test_mode,
 390                                    image_path=self.config['image_path'], version=self.config['version'],
 391                                    host_id='openvim_controller', develop_mode=host_develop_mode,
 392                                    develop_bridge_iface=bridge_ifaces)
 393
 394         self.config['host_threads']['openvim_controller'] = dhcp_host
 395         if not host_test_mode:
 396             dhcp_host.ssh_connect()
 397         return dhcp_host
 398
 399     def launch_dhcp_server(self, vlan, first_ip, last_ip, cidr, gateway):
 400         """
 401         Launch a dhcpserver base on dnsmasq attached to the net base on vlan id across the the openvim computes
 402         :param vlan: vlan identifier
 403         :param first_ip: First dhcp range ip
 404         :param last_ip: Last dhcp range ip
 405         :param cidr: net cidr
 406         :return:
 407         """
 408         ip_tools = IPNetwork(cidr)
 409         dhcp_netmask = str(ip_tools.netmask)
 410         ip_range = [first_ip, last_ip]
 411
 412         dhcp_path = self.config['ovs_controller_file_path']
 413
 414         controller_host = self.get_dhcp_controller()
 415         controller_host.create_linux_bridge(vlan)
 416         controller_host.create_dhcp_interfaces(vlan, first_ip, dhcp_netmask)
 417         controller_host.launch_dhcp_server(vlan, ip_range, dhcp_netmask, dhcp_path, gateway)
 418
 419