RO/osm_ro/openmanod.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   6 # This file is part of openmano
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact with: nfvlabs@tid.es
  23 ##
  24
  25 """
  26 openmano server.
  27 Main program that implements a reference NFVO (Network Functions Virtualisation Orchestrator).
  28 It interfaces with an NFV VIM through its API and offers a northbound interface, based on REST (openmano API),
  29 where NFV services are offered including the creation and deletion of VNF templates, VNF instances,
  30 network service templates and network service instances.
  31
  32 It loads the configuration file and launches the http_server thread that will listen requests using openmano API.
  33 """
  34
  35 import time
  36 import sys
  37 import getopt
  38 import yaml
  39 from os import environ, path as os_path
  40 from jsonschema import validate as js_v, exceptions as js_e
  41 import logging
  42 import logging.handlers as log_handlers
  43 import socket
  44
  45 from yaml import MarkedYAMLError
  46
  47 from osm_ro import httpserver, nfvo, nfvo_db
  48 from osm_ro.openmano_schemas import config_schema
  49 from osm_ro.db_base import db_base_Exception
  50 from osm_ro.wim.engine import WimEngine
  51 from osm_ro.wim.persistence import WimPersistence
  52 import osm_ro
  53
  54 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes"
  55 __date__ = "$26-aug-2014 11:09:29$"
  56 __version__ = "8.0.0.post1"
  57 version_date = "Oct 2020"
  58 database_version = 42      # expected database schema version
  59
  60 global global_config
  61 global logger
  62
  63
  64 class LoadConfigurationException(Exception):
  65     pass
  66
  67
  68 def load_configuration(configuration_file):
  69     default_tokens = {'http_port': 9090,
  70                       'http_host': 'localhost',
  71                       'http_console_proxy': True,
  72                       'http_console_host': None,
  73                       'log_level': 'DEBUG',
  74                       'log_socket_port': 9022,
  75                       'auto_push_VNF_to_VIMs': True,
  76                       'db_host': 'localhost',
  77                       'db_ovim_host': 'localhost'
  78                       }
  79     try:
  80         # Check config file exists
  81         with open(configuration_file, 'r') as f:
  82             config_str = f.read()
  83         # Parse configuration file
  84         config = yaml.load(config_str, Loader=yaml.SafeLoader)
  85         # Validate configuration file with the config_schema
  86         js_v(config, config_schema)
  87
  88         # Add default values tokens
  89         for k, v in default_tokens.items():
  90             if k not in config:
  91                 config[k] = v
  92         return config
  93
  94     except yaml.YAMLError as e:
  95         error_pos = ""
  96         if isinstance(e, MarkedYAMLError):
  97             mark = e.problem_mark
  98             error_pos = " at line:{} column:{}".format(mark.line + 1, mark.column + 1)
  99         raise LoadConfigurationException("Bad YAML format at configuration file '{file}'{pos}: {message}".format(
 100             file=configuration_file, pos=error_pos, message=e))
 101     except js_e.ValidationError as e:
 102         error_pos = ""
 103         if e.path:
 104             error_pos = " at '" + ":".join(map(str, e.path)) + "'"
 105         raise LoadConfigurationException("Invalid field at configuration file '{file}'{pos} {message}".format(
 106             file=configuration_file, pos=error_pos, message=e))
 107     except Exception as e:
 108         raise LoadConfigurationException("Cannot load configuration file '{file}' {message}".format(
 109             file=configuration_file, message=e))
 110
 111
 112 def console_port_iterator():
 113     """
 114     this iterator deals with the http_console_ports
 115     returning the ports one by one
 116     """
 117     index = 0
 118     while index < len(global_config["http_console_ports"]):
 119         port = global_config["http_console_ports"][index]
 120         if type(port) is int:
 121             yield port
 122         else:  # this is dictionary with from to keys
 123             port2 = port["from"]
 124             while port2 <= port["to"]:
 125                 yield port2
 126                 port2 += 1
 127         index += 1
 128
 129
 130 def usage():
 131     print("Usage: ", sys.argv[0], "[options]")
 132     print("      -v|--version: prints current version")
 133     print("      -c|--config [configuration_file]: loads the configuration file (default: openmanod.cfg)")
 134     print("      -h|--help: shows this help")
 135     print(
 136         "      -p|--port [port_number]: changes port number and overrides the port number in the configuration file (default: 9090)")
 137     print(
 138         "      -P|--adminport [port_number]: changes admin port number and overrides the port number in the configuration file (default: 9095)")
 139     print("      --log-socket-host HOST: send logs to this host")
 140     print("      --log-socket-port PORT: send logs using this port (default: 9022)")
 141     print("      --log-file FILE: send logs to this file")
 142     print(
 143         "      --create-tenant NAME: Try to creates this tenant name before starting, ignoring any errors as e.g. conflict")
 144     return
 145
 146
 147 def set_logging_file(log_file):
 148     try:
 149         file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=100e6, backupCount=9, delay=0)
 150         file_handler.setFormatter(log_formatter_simple)
 151         logger.addHandler(file_handler)
 152         # remove initial stream handler
 153         logging.root.removeHandler(logging.root.handlers[0])
 154         print ("logging on '{}'".format(log_file))
 155     except IOError as e:
 156         raise LoadConfigurationException(
 157             "Cannot open logging file '{}': {}. Check folder exist and permissions".format(log_file, e))
 158
 159
 160 def _get_version():
 161     """
 162     Try to get version from package using pkg_resources (available with setuptools)
 163     """
 164     global __version__
 165     ro_version = __version__
 166     try:
 167         from pkg_resources import get_distribution
 168         ro_version = get_distribution("osm_ro").version
 169     except Exception:
 170         pass
 171     return ro_version
 172
 173
 174 if __name__ == "__main__":
 175     log_modules = ("nfvo", "http", "vim", "wim", "db", "console", "ovim", "sdn", "sdnconn")
 176     # env2config contains environ variable names and the correspondence with configuration file openmanod.cfg keys.
 177     # If this environ is defined, this value is taken instead of the one at at configuration file
 178     env2config = {
 179         'RO_DB_HOST': 'db_host',
 180         'RO_DB_NAME': 'db_name',
 181         'RO_DB_USER': 'db_user',
 182         'RO_DB_PASSWORD': 'db_passwd',
 183         'RO_DB_OVIM_HOST': 'db_ovim_host',
 184         'RO_DB_OVIM_NAME': 'db_ovim_name',
 185         'RO_DB_OVIM_USER': 'db_ovim_user',
 186         'RO_DB_OVIM_PASSWORD': 'db_ovim_passwd',
 187         'RO_LOG_LEVEL': 'log_level',
 188         'RO_LOG_FILE': 'log_file',
 189     }
 190     for log_module in log_modules:
 191         env2config['RO_LOG_LEVEL_' + log_module.upper()] = 'log_level_' + log_module
 192     ro_version = _get_version()
 193     # Configure logging step 1
 194     hostname = socket.gethostname()
 195     log_formatter_str = '%(asctime)s.%(msecs)03d00Z[{host}@openmanod] %(filename)s:%(lineno)s severity:%(levelname)s logger:%(name)s log:%(message)s'
 196     log_formatter_complete = logging.Formatter(log_formatter_str.format(host=hostname), datefmt='%Y-%m-%dT%H:%M:%S')
 197     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(thread)d %(filename)s:%(lineno)s %(message)s"
 198     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S.%03d')
 199     logging.basicConfig(format=log_format_simple, level=logging.DEBUG, datefmt='%Y-%m-%dT%H:%M:%S.%03d')
 200     logger = logging.getLogger('openmano')
 201     logger.setLevel(logging.DEBUG)
 202     socket_handler = None
 203     # Read parameters and configuration file
 204     httpthread = None
 205     try:
 206         # load parameters and configuration
 207         opts, args = getopt.getopt(sys.argv[1:], "hvc:V:p:P:",
 208                                    ["config=", "help", "version", "port=", "vnf-repository=", "adminport=",
 209                                     "log-socket-host=", "log-socket-port=", "log-file=", "create-tenant="])
 210         port = None
 211         port_admin = None
 212         config_file = 'openmanod.cfg'
 213         vnf_repository = None
 214         log_file = None
 215         log_socket_host = None
 216         log_socket_port = None
 217         create_tenant = None
 218
 219         for o, a in opts:
 220             if o in ("-v", "--version"):
 221                 print ("openmanod version {} {}".format(ro_version, version_date))
 222                 print ("(c) Copyright Telefonica")
 223                 sys.exit()
 224             elif o in ("-h", "--help"):
 225                 usage()
 226                 sys.exit()
 227             elif o in ("-V", "--vnf-repository"):
 228                 vnf_repository = a
 229             elif o in ("-c", "--config"):
 230                 config_file = a
 231             elif o in ("-p", "--port"):
 232                 port = a
 233             elif o in ("-P", "--adminport"):
 234                 port_admin = a
 235             elif o == "--log-socket-port":
 236                 log_socket_port = a
 237             elif o == "--log-socket-host":
 238                 log_socket_host = a
 239             elif o == "--log-file":
 240                 log_file = a
 241             elif o == "--create-tenant":
 242                 create_tenant = a
 243             else:
 244                 assert False, "Unhandled option"
 245         if log_file:
 246             set_logging_file(log_file)
 247         global_config = load_configuration(config_file)
 248         global_config["version"] = ro_version
 249         global_config["version_date"] = version_date
 250         # Override parameters obtained by command line on ENV
 251         if port:
 252             global_config['http_port'] = port
 253         if port_admin:
 254             global_config['http_admin_port'] = port_admin
 255         if log_socket_host:
 256             global_config['log_socket_host'] = log_socket_host
 257         if log_socket_port:
 258             global_config['log_socket_port'] = log_socket_port
 259
 260         # override with ENV
 261         for env_k, env_v in environ.items():
 262             try:
 263                 if not env_k.startswith("RO_") or env_k not in env2config or not env_v:
 264                     continue
 265                 global_config[env2config[env_k]] = env_v
 266                 if env_k.endswith("PORT"):  # convert to int, skip if not possible
 267                     global_config[env2config[env_k]] = int(env_v)
 268             except Exception as e:
 269                 logger.warning("skipping environ '{}={}' because exception '{}'".format(env_k, env_v, e))
 270
 271         global_config["console_port_iterator"] = console_port_iterator
 272         global_config["console_thread"] = {}
 273         global_config["console_ports"] = {}
 274         if not global_config["http_console_host"]:
 275             global_config["http_console_host"] = global_config["http_host"]
 276             if global_config["http_host"] == "0.0.0.0":
 277                 global_config["http_console_host"] = socket.gethostname()
 278
 279         # Configure logging STEP 2
 280         if "log_host" in global_config:
 281             socket_handler = log_handlers.SocketHandler(global_config["log_socket_host"],
 282                                                         global_config["log_socket_port"])
 283             socket_handler.setFormatter(log_formatter_complete)
 284             if global_config.get("log_socket_level") \
 285                     and global_config["log_socket_level"] != global_config["log_level"]:
 286                 socket_handler.setLevel(global_config["log_socket_level"])
 287             logger.addHandler(socket_handler)
 288
 289         if log_file:
 290             global_config['log_file'] = log_file
 291         elif global_config.get('log_file'):
 292             set_logging_file(global_config['log_file'])
 293
 294         logger.setLevel(getattr(logging, global_config['log_level']))
 295         logger.critical("Starting openmano server version: '%s %s' command: '%s'",
 296                         ro_version, version_date, " ".join(sys.argv))
 297
 298         for log_module in log_modules:
 299             log_level_module = "log_level_" + log_module
 300             log_file_module = "log_file_" + log_module
 301             logger_module = logging.getLogger('openmano.' + log_module)
 302             if log_level_module in global_config:
 303                 logger_module.setLevel(global_config[log_level_module])
 304             if log_file_module in global_config:
 305                 try:
 306                     file_handler = logging.handlers.RotatingFileHandler(global_config[log_file_module],
 307                                                                         maxBytes=100e6, backupCount=9, delay=0)
 308                     file_handler.setFormatter(log_formatter_simple)
 309                     logger_module.addHandler(file_handler)
 310                 except IOError as e:
 311                     raise LoadConfigurationException(
 312                         "Cannot open logging file '{}': {}. Check folder exist and permissions".format(
 313                             global_config[log_file_module], str(e)))
 314             global_config["logger_" + log_module] = logger_module
 315
 316         # Initialize DB connection
 317         mydb = nfvo_db.nfvo_db()
 318         mydb.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'],
 319                      global_config['db_name'])
 320         db_path = osm_ro.__path__[0] + "/database_utils"
 321         if not os_path.exists(db_path + "/migrate_mano_db.sh"):
 322             db_path = osm_ro.__path__[0] + "/../database_utils"
 323         try:
 324             r = mydb.get_db_version()
 325             if r[0] != database_version:
 326                 logger.critical("DATABASE wrong version '{current}'. Try to upgrade/downgrade to version '{target}'"
 327                                 " with '{db_path}/migrate_mano_db.sh {target}'".format(current=r[0],
 328                                                                                        target=database_version,
 329                                                                                        db_path=db_path))
 330                 exit(-1)
 331         except db_base_Exception as e:
 332             logger.critical("DATABASE is not valid. If you think it is corrupted, you can init it with"
 333                             " '{db_path}/init_mano_db.sh' script".format(db_path=db_path))
 334             exit(-1)
 335
 336         nfvo.global_config = global_config
 337         if create_tenant:
 338             try:
 339                 nfvo.new_tenant(mydb, {"name": create_tenant})
 340             except Exception as e:
 341                 if isinstance(e, nfvo.NfvoException) and e.http_code == 409:
 342                     pass  # if tenant exist (NfvoException error 409), ignore
 343                 else:  # otherwise print and error and continue
 344                     logger.error("Cannot create tenant '{}': {}".format(create_tenant, e))
 345
 346         # WIM module
 347         wim_persistence = WimPersistence(mydb)
 348         wim_engine = WimEngine(wim_persistence, nfvo.plugins)
 349         # ---
 350         nfvo.start_service(mydb, wim_persistence, wim_engine)
 351
 352         httpthread = httpserver.httpserver(
 353             mydb, False,
 354             global_config['http_host'], global_config['http_port'],
 355             wim_persistence, wim_engine
 356         )
 357
 358         httpthread.start()
 359         if 'http_admin_port' in global_config:
 360             httpthreadadmin = httpserver.httpserver(mydb, True, global_config['http_host'],
 361                                                     global_config['http_admin_port'])
 362             httpthreadadmin.start()
 363         time.sleep(1)
 364         logger.info('Waiting for http clients')
 365         print('Waiting for http clients')
 366         print('openmanod ready')
 367         print('====================')
 368         time.sleep(20)
 369         sys.stdout.flush()
 370
 371         # TODO: Interactive console must be implemented here instead of join or sleep
 372
 373         # httpthread.join()
 374         # if 'http_admin_port' in global_config:
 375         #    httpthreadadmin.join()
 376         while True:
 377             time.sleep(86400)
 378
 379     except KeyboardInterrupt as e:
 380         logger.info(str(e))
 381     except SystemExit:
 382         pass
 383     except getopt.GetoptError as e:
 384         logger.critical(str(e))  # will print something like "option -a not recognized"
 385         exit(-1)
 386     except LoadConfigurationException as e:
 387         logger.critical(str(e))
 388         exit(-1)
 389     except db_base_Exception as e:
 390         logger.critical(str(e))
 391         exit(-1)
 392     except nfvo.NfvoException as e:
 393         logger.critical(str(e), exc_info=True)
 394         exit(-1)
 395     nfvo.stop_service()
 396     if httpthread:
 397         httpthread.join(1)