RO/osm_ro/openmanod.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   6 # This file is part of openmano
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact with: nfvlabs@tid.es
  23 ##
  24
  25 """
  26 openmano server.
  27 Main program that implements a reference NFVO (Network Functions Virtualisation Orchestrator).
  28 It interfaces with an NFV VIM through its API and offers a northbound interface, based on REST (openmano API),
  29 where NFV services are offered including the creation and deletion of VNF templates, VNF instances,
  30 network service templates and network service instances.
  31
  32 It loads the configuration file and launches the http_server thread that will listen requests using openmano API.
  33 """
  34
  35 import time
  36 import sys
  37 import getopt
  38 import yaml
  39 from os import environ, path as os_path
  40 from jsonschema import validate as js_v, exceptions as js_e
  41 import logging
  42 import logging.handlers as log_handlers
  43 import socket
  44
  45 from yaml import MarkedYAMLError
  46
  47 from osm_ro import httpserver, nfvo, nfvo_db
  48 from osm_ro.openmano_schemas import config_schema
  49 from osm_ro.db_base import db_base_Exception
  50 from osm_ro.wim.engine import WimEngine
  51 from osm_ro.wim.persistence import WimPersistence
  52 import osm_ro
  53
  54 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes"
  55 __date__ = "$26-aug-2014 11:09:29$"
  56 __version__ = "6.0.4.post6"
  57 version_date = "Oct 2019"
  58 database_version = 40      # expected database schema version
  59
  60 global global_config
  61 global logger
  62
  63
  64 class LoadConfigurationException(Exception):
  65     pass
  66
  67
  68 def load_configuration(configuration_file):
  69     default_tokens = {'http_port': 9090,
  70                       'http_host': 'localhost',
  71                       'http_console_proxy': True,
  72                       'http_console_host': None,
  73                       'log_level': 'DEBUG',
  74                       'log_socket_port': 9022,
  75                       'auto_push_VNF_to_VIMs': True,
  76                       'db_host': 'localhost',
  77                       'db_ovim_host': 'localhost'
  78                       }
  79     try:
  80         # Check config file exists
  81         with open(configuration_file, 'r') as f:
  82             config_str = f.read()
  83         # Parse configuration file
  84         config = yaml.load(config_str, Loader=yaml.SafeLoader)
  85         # Validate configuration file with the config_schema
  86         js_v(config, config_schema)
  87
  88         # Add default values tokens
  89         for k, v in default_tokens.items():
  90             if k not in config:
  91                 config[k] = v
  92         return config
  93
  94     except yaml.YAMLError as e:
  95         error_pos = ""
  96         if isinstance(e, MarkedYAMLError):
  97             mark = e.problem_mark
  98             error_pos = " at line:{} column:{}".format(mark.line + 1, mark.column + 1)
  99         raise LoadConfigurationException("Bad YAML format at configuration file '{file}'{pos}: {message}".format(
 100             file=configuration_file, pos=error_pos, message=e))
 101     except js_e.ValidationError as e:
 102         error_pos = ""
 103         if e.path:
 104             error_pos = " at '" + ":".join(map(str, e.path)) + "'"
 105         raise LoadConfigurationException("Invalid field at configuration file '{file}'{pos} {message}".format(
 106             file=configuration_file, pos=error_pos, message=e))
 107     except Exception as e:
 108         raise LoadConfigurationException("Cannot load configuration file '{file}' {message}".format(
 109             file=configuration_file, message=e))
 110
 111
 112 def console_port_iterator():
 113     """
 114     this iterator deals with the http_console_ports
 115     returning the ports one by one
 116     """
 117     index = 0
 118     while index < len(global_config["http_console_ports"]):
 119         port = global_config["http_console_ports"][index]
 120         if type(port) is int:
 121             yield port
 122         else:  # this is dictionary with from to keys
 123             port2 = port["from"]
 124             while port2 <= port["to"]:
 125                 yield port2
 126                 port2 += 1
 127         index += 1
 128
 129
 130 def usage():
 131     print("Usage: ", sys.argv[0], "[options]")
 132     print("      -v|--version: prints current version")
 133     print("      -c|--config [configuration_file]: loads the configuration file (default: openmanod.cfg)")
 134     print("      -h|--help: shows this help")
 135     print(
 136         "      -p|--port [port_number]: changes port number and overrides the port number in the configuration file (default: 9090)")
 137     print(
 138         "      -P|--adminport [port_number]: changes admin port number and overrides the port number in the configuration file (default: 9095)")
 139     print("      --log-socket-host HOST: send logs to this host")
 140     print("      --log-socket-port PORT: send logs using this port (default: 9022)")
 141     print("      --log-file FILE: send logs to this file")
 142     print(
 143         "      --create-tenant NAME: Try to creates this tenant name before starting, ignoring any errors as e.g. conflict")
 144     return
 145
 146
 147 def set_logging_file(log_file):
 148     try:
 149         file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=100e6, backupCount=9, delay=0)
 150         file_handler.setFormatter(log_formatter_simple)
 151         logger.addHandler(file_handler)
 152         # remove initial stream handler
 153         logging.root.removeHandler(logging.root.handlers[0])
 154         print ("logging on '{}'".format(log_file))
 155     except IOError as e:
 156         raise LoadConfigurationException(
 157             "Cannot open logging file '{}': {}. Check folder exist and permissions".format(log_file, e))
 158
 159
 160 if __name__ == "__main__":
 161     # env2config contains environ variable names and the correspondence with configuration file openmanod.cfg keys.
 162     # If this environ is defined, this value is taken instead of the one at at configuration file
 163     env2config = {
 164         'RO_DB_HOST': 'db_host',
 165         'RO_DB_NAME': 'db_name',
 166         'RO_DB_USER': 'db_user',
 167         'RO_DB_PASSWORD': 'db_passwd',
 168         'RO_DB_OVIM_HOST': 'db_ovim_host',
 169         'RO_DB_OVIM_NAME': 'db_ovim_name',
 170         'RO_DB_OVIM_USER': 'db_ovim_user',
 171         'RO_DB_OVIM_PASSWORD': 'db_ovim_passwd',
 172         'RO_LOG_LEVEL': 'log_level',
 173         'RO_LOG_FILE': 'log_file',
 174     }
 175     # Configure logging step 1
 176     hostname = socket.gethostname()
 177     log_formatter_str = '%(asctime)s.%(msecs)03d00Z[{host}@openmanod] %(filename)s:%(lineno)s severity:%(levelname)s logger:%(name)s log:%(message)s'
 178     log_formatter_complete = logging.Formatter(log_formatter_str.format(host=hostname), datefmt='%Y-%m-%dT%H:%M:%S')
 179     log_format_simple = "%(asctime)s %(levelname)s  %(name)s %(thread)d %(filename)s:%(lineno)s %(message)s"
 180     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 181     logging.basicConfig(format=log_format_simple, level=logging.DEBUG)
 182     logger = logging.getLogger('openmano')
 183     logger.setLevel(logging.DEBUG)
 184     socket_handler = None
 185     # Read parameters and configuration file
 186     httpthread = None
 187     try:
 188         # load parameters and configuration
 189         opts, args = getopt.getopt(sys.argv[1:], "hvc:V:p:P:",
 190                                    ["config=", "help", "version", "port=", "vnf-repository=", "adminport=",
 191                                     "log-socket-host=", "log-socket-port=", "log-file=", "create-tenant="])
 192         port = None
 193         port_admin = None
 194         config_file = 'openmanod.cfg'
 195         vnf_repository = None
 196         log_file = None
 197         log_socket_host = None
 198         log_socket_port = None
 199         create_tenant = None
 200
 201         for o, a in opts:
 202             if o in ("-v", "--version"):
 203                 print ("openmanod version " + __version__ + ' ' + version_date)
 204                 print ("(c) Copyright Telefonica")
 205                 sys.exit()
 206             elif o in ("-h", "--help"):
 207                 usage()
 208                 sys.exit()
 209             elif o in ("-V", "--vnf-repository"):
 210                 vnf_repository = a
 211             elif o in ("-c", "--config"):
 212                 config_file = a
 213             elif o in ("-p", "--port"):
 214                 port = a
 215             elif o in ("-P", "--adminport"):
 216                 port_admin = a
 217             elif o == "--log-socket-port":
 218                 log_socket_port = a
 219             elif o == "--log-socket-host":
 220                 log_socket_host = a
 221             elif o == "--log-file":
 222                 log_file = a
 223             elif o == "--create-tenant":
 224                 create_tenant = a
 225             else:
 226                 assert False, "Unhandled option"
 227         if log_file:
 228             set_logging_file(log_file)
 229         global_config = load_configuration(config_file)
 230         global_config["version"] = __version__
 231         global_config["version_date"] = version_date
 232         # Override parameters obtained by command line on ENV
 233         if port:
 234             global_config['http_port'] = port
 235         if port_admin:
 236             global_config['http_admin_port'] = port_admin
 237         if log_socket_host:
 238             global_config['log_socket_host'] = log_socket_host
 239         if log_socket_port:
 240             global_config['log_socket_port'] = log_socket_port
 241
 242         # override with ENV
 243         for env_k, env_v in environ.items():
 244             try:
 245                 if not env_k.startswith("RO_") or env_k not in env2config or not env_v:
 246                     continue
 247                 global_config[env2config[env_k]] = env_v
 248                 if env_k.endswith("PORT"):  # convert to int, skip if not possible
 249                     global_config[env2config[env_k]] = int(env_v)
 250             except Exception as e:
 251                 logger.warn("skipping environ '{}={}' because exception '{}'".format(env_k, env_v, e))
 252
 253         global_config["console_port_iterator"] = console_port_iterator
 254         global_config["console_thread"] = {}
 255         global_config["console_ports"] = {}
 256         if not global_config["http_console_host"]:
 257             global_config["http_console_host"] = global_config["http_host"]
 258             if global_config["http_host"] == "0.0.0.0":
 259                 global_config["http_console_host"] = socket.gethostname()
 260
 261         # Configure logging STEP 2
 262         if "log_host" in global_config:
 263             socket_handler = log_handlers.SocketHandler(global_config["log_socket_host"],
 264                                                         global_config["log_socket_port"])
 265             socket_handler.setFormatter(log_formatter_complete)
 266             if global_config.get("log_socket_level") \
 267                     and global_config["log_socket_level"] != global_config["log_level"]:
 268                 socket_handler.setLevel(global_config["log_socket_level"])
 269             logger.addHandler(socket_handler)
 270
 271         if log_file:
 272             global_config['log_file'] = log_file
 273         elif global_config.get('log_file'):
 274             set_logging_file(global_config['log_file'])
 275
 276         logger.setLevel(getattr(logging, global_config['log_level']))
 277         logger.critical("Starting openmano server version: '%s %s' command: '%s'",
 278                         __version__, version_date, " ".join(sys.argv))
 279
 280         for log_module in ("nfvo", "http", "vim", "wim", "db", "console", "ovim"):
 281             log_level_module = "log_level_" + log_module
 282             log_file_module = "log_file_" + log_module
 283             logger_module = logging.getLogger('openmano.' + log_module)
 284             if log_level_module in global_config:
 285                 logger_module.setLevel(global_config[log_level_module])
 286             if log_file_module in global_config:
 287                 try:
 288                     file_handler = logging.handlers.RotatingFileHandler(global_config[log_file_module],
 289                                                                         maxBytes=100e6, backupCount=9, delay=0)
 290                     file_handler.setFormatter(log_formatter_simple)
 291                     logger_module.addHandler(file_handler)
 292                 except IOError as e:
 293                     raise LoadConfigurationException(
 294                         "Cannot open logging file '{}': {}. Check folder exist and permissions".format(
 295                             global_config[log_file_module], str(e)))
 296             global_config["logger_" + log_module] = logger_module
 297
 298         # Initialize DB connection
 299         mydb = nfvo_db.nfvo_db()
 300         mydb.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'],
 301                      global_config['db_name'])
 302         db_path = osm_ro.__path__[0] + "/database_utils"
 303         if not os_path.exists(db_path + "/migrate_mano_db.sh"):
 304             db_path = osm_ro.__path__[0] + "/../database_utils"
 305         try:
 306             r = mydb.get_db_version()
 307             if r[0] != database_version:
 308                 logger.critical("DATABASE wrong version '{current}'. Try to upgrade/downgrade to version '{target}'"
 309                                 " with '{db_path}/migrate_mano_db.sh {target}'".format(current=r[0],
 310                                                                                        target=database_version,
 311                                                                                        db_path=db_path))
 312                 exit(-1)
 313         except db_base_Exception as e:
 314             logger.critical("DATABASE is not valid. If you think it is corrupted, you can init it with"
 315                             " '{db_path}/init_mano_db.sh' script".format(db_path=db_path))
 316             exit(-1)
 317
 318         nfvo.global_config = global_config
 319         if create_tenant:
 320             try:
 321                 nfvo.new_tenant(mydb, {"name": create_tenant})
 322             except Exception as e:
 323                 if isinstance(e, nfvo.NfvoException) and e.http_code == 409:
 324                     pass  # if tenant exist (NfvoException error 409), ignore
 325                 else:  # otherwise print and error and continue
 326                     logger.error("Cannot create tenant '{}': {}".format(create_tenant, e))
 327
 328         # WIM module
 329         wim_persistence = WimPersistence(mydb)
 330         wim_engine = WimEngine(wim_persistence, nfvo.plugins)
 331         # ---
 332         nfvo.start_service(mydb, wim_persistence, wim_engine)
 333
 334         httpthread = httpserver.httpserver(
 335             mydb, False,
 336             global_config['http_host'], global_config['http_port'],
 337             wim_persistence, wim_engine
 338         )
 339
 340         httpthread.start()
 341         if 'http_admin_port' in global_config:
 342             httpthreadadmin = httpserver.httpserver(mydb, True, global_config['http_host'],
 343                                                     global_config['http_admin_port'])
 344             httpthreadadmin.start()
 345         time.sleep(1)
 346         logger.info('Waiting for http clients')
 347         print('Waiting for http clients')
 348         print('openmanod ready')
 349         print('====================')
 350         time.sleep(20)
 351         sys.stdout.flush()
 352
 353         # TODO: Interactive console must be implemented here instead of join or sleep
 354
 355         # httpthread.join()
 356         # if 'http_admin_port' in global_config:
 357         #    httpthreadadmin.join()
 358         while True:
 359             time.sleep(86400)
 360
 361     except KeyboardInterrupt as e:
 362         logger.info(str(e))
 363     except SystemExit:
 364         pass
 365     except getopt.GetoptError as e:
 366         logger.critical(str(e))  # will print something like "option -a not recognized"
 367         exit(-1)
 368     except LoadConfigurationException as e:
 369         logger.critical(str(e))
 370         exit(-1)
 371     except db_base_Exception as e:
 372         logger.critical(str(e))
 373         exit(-1)
 374     except nfvo.NfvoException as e:
 375         logger.critical(str(e), exc_info=True)
 376         exit(-1)
 377     nfvo.stop_service()
 378     if httpthread:
 379         httpthread.join(1)