openvimd.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   6 # This file is part of openvim
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact with: nfvlabs@tid.es
  23 ##
  24
  25 '''
  26 This is the main program of openvim, it reads the configuration
  27 and launches the rest of threads: http clients, openflow controller
  28 and host controllers
  29 '''
  30
  31 __author__="Alfonso Tierno"
  32 __date__ ="$10-jul-2014 12:07:15$"
  33 __version__="0.5.2-r519"
  34 version_date="Jan 2017"
  35 database_version="0.9"      #expected database schema version
  36
  37 import httpserver
  38 import auxiliary_functions as af
  39 import sys
  40 import getopt
  41 import time
  42 import vim_db
  43 import yaml
  44 import os
  45 from jsonschema import validate as js_v, exceptions as js_e
  46 import host_thread as ht
  47 import dhcp_thread as dt
  48 import openflow_thread as oft
  49 import threading
  50 from vim_schema import config_schema
  51 import logging
  52 import logging.handlers as log_handlers
  53 import imp
  54 import socket
  55
  56 global config_dic
  57 global logger
  58 logger = logging.getLogger('vim')
  59
  60 class LoadConfigurationException(Exception):
  61     pass
  62
  63 def load_configuration(configuration_file):
  64     default_tokens ={'http_port':9080, 'http_host':'localhost',
  65                      'of_controller_nets_with_same_vlan':True,
  66                      'image_path':'/opt/VNF/images',
  67                      'network_vlan_range_start':1000,
  68                      'network_vlan_range_end': 4096,
  69                      'log_level': "DEBUG",
  70                      'log_level_db': "ERROR",
  71                      'log_level_of': 'ERROR',
  72                      'bridge_ifaces': {},
  73                      'network_type': 'ovs',
  74                      'ovs_controller_user': 'osm_dhcp',
  75                      'ovs_controller_file_path': '/var/lib/',
  76             }
  77     try:
  78         #First load configuration from configuration file
  79         #Check config file exists
  80         if not os.path.isfile(configuration_file):
  81             return (False, "Configuration file '"+configuration_file+"' does not exists")
  82
  83         #Read and parse file
  84         (return_status, code) = af.read_file(configuration_file)
  85         if not return_status:
  86             return (return_status, "Error loading configuration file '"+configuration_file+"': "+code)
  87         try:
  88             config = yaml.load(code)
  89         except yaml.YAMLError, exc:
  90             error_pos = ""
  91             if hasattr(exc, 'problem_mark'):
  92                 mark = exc.problem_mark
  93                 error_pos = " at position: (%s:%s)" % (mark.line+1, mark.column+1)
  94             return (False, "Error loading configuration file '"+configuration_file+"'"+error_pos+": content format error: Failed to parse yaml format")
  95
  96
  97         try:
  98             js_v(config, config_schema)
  99         except js_e.ValidationError, exc:
 100             error_pos = ""
 101             if len(exc.path)>0: error_pos=" at '" + ":".join(map(str, exc.path))+"'"
 102             return False, "Error loading configuration file '"+configuration_file+"'"+error_pos+": "+exc.message
 103
 104
 105         #Check default values tokens
 106         for k,v in default_tokens.items():
 107             if k not in config: config[k]=v
 108         #Check vlan ranges
 109         if config["network_vlan_range_start"]+10 >= config["network_vlan_range_end"]:
 110             return False, "Error invalid network_vlan_range less than 10 elements"
 111
 112     except Exception,e:
 113         return (False, "Error loading configuration file '"+configuration_file+"': "+str(e))
 114     return (True, config)
 115
 116 def create_database_connection(config_dic):
 117     db = vim_db.vim_db( (config_dic["network_vlan_range_start"],config_dic["network_vlan_range_end"]), config_dic['log_level_db'] );
 118     if db.connect(config_dic['db_host'], config_dic['db_user'], config_dic['db_passwd'], config_dic['db_name']) == -1:
 119         logger.error("Cannot connect to database %s at %s@%s", config_dic['db_name'], config_dic['db_user'], config_dic['db_host'])
 120         exit(-1)
 121     return db
 122
 123 def usage():
 124     print "Usage: ", sys.argv[0], "[options]"
 125     print "      -v|--version: prints current version"
 126     print "      -c|--config FILE: loads the configuration file (default: openvimd.cfg)"
 127     print "      -h|--help: shows this help"
 128     print "      -p|--port PORT: changes port number and overrides the port number in the configuration file (default: 908)"
 129     print "      -P|--adminport PORT: changes admin port number and overrides the port number in the configuration file (default: not listen)"
 130     print "      --dbname NAME: changes db_name and overrides the db_name in the configuration file"
 131     #print( "      --log-socket-host HOST: send logs to this host")
 132     #print( "      --log-socket-port PORT: send logs using this port (default: 9022)")
 133     print( "      --log-file FILE: send logs to this file")
 134     return
 135
 136
 137 if __name__=="__main__":
 138     hostname = socket.gethostname()
 139     #streamformat = "%(levelname)s (%(module)s:%(lineno)d) %(message)s"
 140     log_formatter_complete = logging.Formatter(
 141         '%(asctime)s.%(msecs)03d00Z[{host}@openmanod] %(filename)s:%(lineno)s severity:%(levelname)s logger:%(name)s log:%(message)s'.format(host=hostname),
 142         datefmt='%Y-%m-%dT%H:%M:%S',
 143     )
 144     log_format_simple =  "%(asctime)s %(levelname)s  %(name)s %(filename)s:%(lineno)s %(message)s"
 145     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 146     logging.basicConfig(format=log_format_simple, level= logging.DEBUG)
 147     logger = logging.getLogger('openmano')
 148     logger.setLevel(logging.DEBUG)
 149     try:
 150         opts, args = getopt.getopt(sys.argv[1:], "hvc:p:P:", ["config=", "help", "version", "port=", "adminport=", "log-file=", "dbname="])
 151     except getopt.GetoptError, err:
 152         # print help information and exit:
 153         logger.error("%s. Type -h for help", err) # will print something like "option -a not recognized"
 154         #usage()
 155         sys.exit(-2)
 156
 157     port=None
 158     port_admin = None
 159     config_file = 'openvimd.cfg'
 160     log_file = None
 161     db_name = None
 162
 163     for o, a in opts:
 164         if o in ("-v", "--version"):
 165             print "openvimd version", __version__, version_date
 166             print "(c) Copyright Telefonica"
 167             sys.exit(0)
 168         elif o in ("-h", "--help"):
 169             usage()
 170             sys.exit(0)
 171         elif o in ("-c", "--config"):
 172             config_file = a
 173         elif o in ("-p", "--port"):
 174             port = a
 175         elif o in ("-P", "--adminport"):
 176             port_admin = a
 177         elif o in ("-P", "--dbname"):
 178             db_name = a
 179         elif o == "--log-file":
 180             log_file = a
 181         else:
 182             assert False, "Unhandled option"
 183
 184
 185     try:
 186         #Load configuration file
 187         r, config_dic = load_configuration(config_file)
 188         #print config_dic
 189         if not r:
 190             logger.error(config_dic)
 191             config_dic={}
 192             exit(-1)
 193         if log_file:
 194             try:
 195                 file_handler= logging.handlers.RotatingFileHandler(log_file, maxBytes=100e6, backupCount=9, delay=0)
 196                 file_handler.setFormatter(log_formatter_simple)
 197                 logger.addHandler(file_handler)
 198                 #logger.debug("moving logs to '%s'", global_config["log_file"])
 199                 #remove initial stream handler
 200                 logging.root.removeHandler(logging.root.handlers[0])
 201                 print ("logging on '{}'".format(log_file))
 202             except IOError as e:
 203                 raise LoadConfigurationException("Cannot open logging file '{}': {}. Check folder exist and permissions".format(log_file, str(e)) )
 204
 205         logger.setLevel(getattr(logging, config_dic['log_level']))
 206         logger.critical("Starting openvim server command: '%s'", sys.argv[0])
 207         #override parameters obtained by command line
 208         if port:
 209             config_dic['http_port'] = port
 210         if port_admin:
 211             config_dic['http_admin_port'] = port_admin
 212         if db_name:
 213             config_dic['db_name'] = db_name
 214
 215         #check mode
 216         if 'mode' not in config_dic:
 217             config_dic['mode'] = 'normal'
 218             #allow backward compatibility of test_mode option
 219             if 'test_mode' in config_dic and config_dic['test_mode']==True:
 220                 config_dic['mode'] = 'test'
 221         if config_dic['mode'] == 'development' and config_dic['network_type'] == 'bridge' and \
 222                 ( 'development_bridge' not in config_dic or config_dic['development_bridge'] not in config_dic.get("bridge_ifaces",None) ):
 223             logger.error("'%s' is not a valid 'development_bridge', not one of the 'bridge_ifaces'", config_file)
 224             exit(-1)
 225
 226         if config_dic['mode'] != 'normal':
 227             print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
 228             print "!! Warning, openvimd in TEST mode '%s'" % config_dic['mode']
 229             print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
 230         config_dic['version'] = __version__
 231
 232     #Connect to database
 233         db_http = create_database_connection(config_dic)
 234         r = db_http.get_db_version()
 235         if r[0]<0:
 236             logger.error("DATABASE is not a VIM one or it is a '0.0' version. Try to upgrade to version '%s' with './database_utils/migrate_vim_db.sh'", database_version)
 237             exit(-1)
 238         elif r[1]!=database_version:
 239             logger.error("DATABASE wrong version '%s'. Try to upgrade/downgrade to version '%s' with './database_utils/migrate_vim_db.sh'", r[1], database_version)
 240             exit(-1)
 241         db_of = create_database_connection(config_dic)
 242         db_lock= threading.Lock()
 243         config_dic['db'] = db_of
 244         config_dic['db_lock'] = db_lock
 245
 246     #precreate interfaces; [bridge:<host_bridge_name>, VLAN used at Host, uuid of network camping in this bridge, speed in Gbit/s
 247         config_dic['dhcp_nets']=[]
 248         config_dic['bridge_nets']=[]
 249         for bridge,vlan_speed in config_dic["bridge_ifaces"].items():
 250             #skip 'development_bridge'
 251             if config_dic['mode'] == 'development' and config_dic['development_bridge'] == bridge:
 252                 continue
 253             config_dic['bridge_nets'].append( [bridge, vlan_speed[0], vlan_speed[1], None] )
 254         del config_dic["bridge_ifaces"]
 255
 256         #check if this bridge is already used (present at database) for a network)
 257         used_bridge_nets=[]
 258         for brnet in config_dic['bridge_nets']:
 259             r,nets = db_of.get_table(SELECT=('uuid',), FROM='nets',WHERE={'provider': "bridge:"+brnet[0]})
 260             if r>0:
 261                 brnet[3] = nets[0]['uuid']
 262                 used_bridge_nets.append(brnet[0])
 263                 if config_dic.get("dhcp_server"):
 264                     if brnet[0] in config_dic["dhcp_server"]["bridge_ifaces"]:
 265                         config_dic['dhcp_nets'].append(nets[0]['uuid'])
 266         if len(used_bridge_nets) > 0 :
 267             logger.info("found used bridge nets: " + ",".join(used_bridge_nets))
 268         #get nets used by dhcp
 269         if config_dic.get("dhcp_server"):
 270             for net in config_dic["dhcp_server"].get("nets", () ):
 271                 r,nets = db_of.get_table(SELECT=('uuid',), FROM='nets',WHERE={'name': net})
 272                 if r>0:
 273                     config_dic['dhcp_nets'].append(nets[0]['uuid'])
 274
 275     # get host list from data base before starting threads
 276         r,hosts = db_of.get_table(SELECT=('name','ip_name','user','uuid'), FROM='hosts', WHERE={'status':'ok'})
 277         if r<0:
 278             logger.error("Cannot get hosts from database %s", hosts)
 279             exit(-1)
 280     # create connector to the openflow controller
 281         of_test_mode = False if config_dic['mode']=='normal' or config_dic['mode']=="OF only" else True
 282
 283         if of_test_mode:
 284             OF_conn = oft.of_test_connector({"of_debug": config_dic['log_level_of']} )
 285         else:
 286             #load other parameters starting by of_ from config dict in a temporal dict
 287             temp_dict={ "of_ip":  config_dic['of_controller_ip'],
 288                         "of_port": config_dic['of_controller_port'],
 289                         "of_dpid": config_dic['of_controller_dpid'],
 290                         "of_debug":   config_dic['log_level_of']
 291                 }
 292             for k,v in config_dic.iteritems():
 293                 if type(k) is str and k[0:3]=="of_" and k[0:13] != "of_controller":
 294                     temp_dict[k]=v
 295             if config_dic['of_controller']=='opendaylight':
 296                 module = "ODL"
 297             elif "of_controller_module" in config_dic:
 298                 module = config_dic["of_controller_module"]
 299             else:
 300                 module = config_dic['of_controller']
 301             module_info=None
 302             try:
 303                 module_info = imp.find_module(module)
 304
 305                 OF_conn = imp.load_module("OF_conn", *module_info)
 306                 try:
 307                     OF_conn = OF_conn.OF_conn(temp_dict)
 308                 except Exception as e:
 309                     logger.error("Cannot open the Openflow controller '%s': %s", type(e).__name__, str(e))
 310                     if module_info and module_info[0]:
 311                         file.close(module_info[0])
 312                     exit(-1)
 313             except (IOError, ImportError) as e:
 314                 if module_info and module_info[0]:
 315                     file.close(module_info[0])
 316                 logger.error("Cannot open openflow controller module '%s'; %s: %s; revise 'of_controller' field of configuration file.", module, type(e).__name__, str(e))
 317                 exit(-1)
 318
 319
 320     #create openflow thread
 321         thread = oft.openflow_thread(OF_conn, of_test=of_test_mode, db=db_of,  db_lock=db_lock,
 322                         pmp_with_same_vlan=config_dic['of_controller_nets_with_same_vlan'],
 323                         debug=config_dic['log_level_of'])
 324         r,c = thread.OF_connector.obtain_port_correspondence()
 325         if r<0:
 326             logger.error("Cannot get openflow information %s", c)
 327             exit()
 328         thread.start()
 329         config_dic['of_thread'] = thread
 330
 331     #create dhcp_server thread
 332         host_test_mode = True if config_dic['mode']=='test' or config_dic['mode']=="OF only" else False
 333         dhcp_params = config_dic.get("dhcp_server")
 334         if dhcp_params:
 335             thread = dt.dhcp_thread(dhcp_params=dhcp_params, test=host_test_mode, dhcp_nets=config_dic["dhcp_nets"], db=db_of,  db_lock=db_lock, debug=config_dic['log_level_of'])
 336             thread.start()
 337             config_dic['dhcp_thread'] = thread
 338
 339
 340     #Create one thread for each host
 341         host_test_mode = True if config_dic['mode']=='test' or config_dic['mode']=="OF only" else False
 342         host_develop_mode = True if config_dic['mode']=='development' else False
 343         host_develop_bridge_iface = config_dic.get('development_bridge', None)
 344         config_dic['host_threads'] = {}
 345         for host in hosts:
 346             host['image_path'] = '/opt/VNF/images/openvim'
 347             thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=db_of, db_lock=db_lock,
 348                     test=host_test_mode, image_path=config_dic['image_path'], version=config_dic['version'],
 349                     host_id=host['uuid'], develop_mode=host_develop_mode, develop_bridge_iface=host_develop_bridge_iface  )
 350             thread.start()
 351             config_dic['host_threads'][ host['uuid'] ] = thread
 352
 353
 354
 355     #Create thread to listen to web requests
 356         http_thread = httpserver.httpserver(db_http, 'http', config_dic['http_host'], config_dic['http_port'], False, config_dic)
 357         http_thread.start()
 358
 359         if 'http_admin_port' in config_dic:
 360             db_http = create_database_connection(config_dic)
 361             http_thread_admin = httpserver.httpserver(db_http, 'http-admin', config_dic['http_host'], config_dic['http_admin_port'], True)
 362             http_thread_admin.start()
 363         else:
 364             http_thread_admin = None
 365         time.sleep(1)
 366         logger.info('Waiting for http clients')
 367         print ('openvimd ready')
 368         print ('====================')
 369         sys.stdout.flush()
 370
 371         #TODO: Interactive console would be nice here instead of join or sleep
 372
 373         r="help" #force print help at the beginning
 374         while True:
 375             if r=='exit':
 376                 break
 377             elif r!='':
 378                 print "type 'exit' for terminate"
 379             r = raw_input('> ')
 380
 381     except (KeyboardInterrupt, SystemExit):
 382         pass
 383     except SystemExit:
 384         pass
 385     except getopt.GetoptError as e:
 386         logger.critical(str(e)) # will print something like "option -a not recognized"
 387         #usage()
 388         exit(-1)
 389     except LoadConfigurationException as e:
 390         logger.critical(str(e))
 391         exit(-1)
 392
 393     logger.info('Exiting openvimd')
 394     threads = config_dic.get('host_threads', {})
 395     if 'of_thread' in config_dic:
 396         threads['of'] = (config_dic['of_thread'])
 397     if 'dhcp_thread' in config_dic:
 398         threads['dhcp'] = (config_dic['dhcp_thread'])
 399
 400     for thread in threads.values():
 401         thread.insert_task("exit")
 402     for thread in threads.values():
 403         thread.join()
 404     #http_thread.join()
 405     #if http_thread_admin is not None:
 406     #http_thread_admin.join()
 407     logger.debug( "bye!")
 408     exit()
 409