RO/osm_ro/openmanod.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   6 # This file is part of openmano
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact with: nfvlabs@tid.es
  23 ##
  24
  25 """
  26 openmano server.
  27 Main program that implements a reference NFVO (Network Functions Virtualisation Orchestrator).
  28 It interfaces with an NFV VIM through its API and offers a northbound interface, based on REST (openmano API),
  29 where NFV services are offered including the creation and deletion of VNF templates, VNF instances,
  30 network service templates and network service instances.
  31
  32 It loads the configuration file and launches the http_server thread that will listen requests using openmano API.
  33 """
  34
  35 import time
  36 import sys
  37 import getopt
  38 import yaml
  39 from os import environ, path as os_path
  40 from jsonschema import validate as js_v, exceptions as js_e
  41 import logging
  42 import logging.handlers as log_handlers
  43 import socket
  44
  45 from yaml import MarkedYAMLError
  46
  47 from osm_ro import httpserver, nfvo, nfvo_db
  48 from osm_ro.openmano_schemas import config_schema
  49 from osm_ro.db_base import db_base_Exception
  50 from osm_ro.wim.engine import WimEngine
  51 from osm_ro.wim.persistence import WimPersistence
  52 import osm_ro
  53
  54 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes"
  55 __date__ = "$26-aug-2014 11:09:29$"
  56 __version__ = "7.0.1.post18"
  57 version_date = "Apr 2020"
  58 database_version = 40      # expected database schema version
  59
  60 global global_config
  61 global logger
  62
  63
  64 class LoadConfigurationException(Exception):
  65     pass
  66
  67
  68 def load_configuration(configuration_file):
  69     default_tokens = {'http_port': 9090,
  70                       'http_host': 'localhost',
  71                       'http_console_proxy': True,
  72                       'http_console_host': None,
  73                       'log_level': 'DEBUG',
  74                       'log_socket_port': 9022,
  75                       'auto_push_VNF_to_VIMs': True,
  76                       'db_host': 'localhost',
  77                       'db_ovim_host': 'localhost'
  78                       }
  79     try:
  80         # Check config file exists
  81         with open(configuration_file, 'r') as f:
  82             config_str = f.read()
  83         # Parse configuration file
  84         config = yaml.load(config_str, Loader=yaml.SafeLoader)
  85         # Validate configuration file with the config_schema
  86         js_v(config, config_schema)
  87
  88         # Add default values tokens
  89         for k, v in default_tokens.items():
  90             if k not in config:
  91                 config[k] = v
  92         return config
  93
  94     except yaml.YAMLError as e:
  95         error_pos = ""
  96         if isinstance(e, MarkedYAMLError):
  97             mark = e.problem_mark
  98             error_pos = " at line:{} column:{}".format(mark.line + 1, mark.column + 1)
  99         raise LoadConfigurationException("Bad YAML format at configuration file '{file}'{pos}: {message}".format(
 100             file=configuration_file, pos=error_pos, message=e))
 101     except js_e.ValidationError as e:
 102         error_pos = ""
 103         if e.path:
 104             error_pos = " at '" + ":".join(map(str, e.path)) + "'"
 105         raise LoadConfigurationException("Invalid field at configuration file '{file}'{pos} {message}".format(
 106             file=configuration_file, pos=error_pos, message=e))
 107     except Exception as e:
 108         raise LoadConfigurationException("Cannot load configuration file '{file}' {message}".format(
 109             file=configuration_file, message=e))
 110
 111
 112 def console_port_iterator():
 113     """
 114     this iterator deals with the http_console_ports
 115     returning the ports one by one
 116     """
 117     index = 0
 118     while index < len(global_config["http_console_ports"]):
 119         port = global_config["http_console_ports"][index]
 120         if type(port) is int:
 121             yield port
 122         else:  # this is dictionary with from to keys
 123             port2 = port["from"]
 124             while port2 <= port["to"]:
 125                 yield port2
 126                 port2 += 1
 127         index += 1
 128
 129
 130 def usage():
 131     print("Usage: ", sys.argv[0], "[options]")
 132     print("      -v|--version: prints current version")
 133     print("      -c|--config [configuration_file]: loads the configuration file (default: openmanod.cfg)")
 134     print("      -h|--help: shows this help")
 135     print(
 136         "      -p|--port [port_number]: changes port number and overrides the port number in the configuration file (default: 9090)")
 137     print(
 138         "      -P|--adminport [port_number]: changes admin port number and overrides the port number in the configuration file (default: 9095)")
 139     print("      --log-socket-host HOST: send logs to this host")
 140     print("      --log-socket-port PORT: send logs using this port (default: 9022)")
 141     print("      --log-file FILE: send logs to this file")
 142     print(
 143         "      --create-tenant NAME: Try to creates this tenant name before starting, ignoring any errors as e.g. conflict")
 144     return
 145
 146
 147 def set_logging_file(log_file):
 148     try:
 149         file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=100e6, backupCount=9, delay=0)
 150         file_handler.setFormatter(log_formatter_simple)
 151         logger.addHandler(file_handler)
 152         # remove initial stream handler
 153         logging.root.removeHandler(logging.root.handlers[0])
 154         print ("logging on '{}'".format(log_file))
 155     except IOError as e:
 156         raise LoadConfigurationException(
 157             "Cannot open logging file '{}': {}. Check folder exist and permissions".format(log_file, e))
 158
 159
 160 def _get_version():
 161     """
 162     Try to get version from package using pkg_resources (available with setuptools)
 163     """
 164     global __version__
 165     ro_version = __version__
 166     try:
 167         from pkg_resources import get_distribution
 168         ro_version = get_distribution("osm_ro").version
 169     except Exception:
 170         pass
 171     return ro_version
 172
 173
 174 if __name__ == "__main__":
 175     # env2config contains environ variable names and the correspondence with configuration file openmanod.cfg keys.
 176     # If this environ is defined, this value is taken instead of the one at at configuration file
 177     env2config = {
 178         'RO_DB_HOST': 'db_host',
 179         'RO_DB_NAME': 'db_name',
 180         'RO_DB_USER': 'db_user',
 181         'RO_DB_PASSWORD': 'db_passwd',
 182         'RO_DB_OVIM_HOST': 'db_ovim_host',
 183         'RO_DB_OVIM_NAME': 'db_ovim_name',
 184         'RO_DB_OVIM_USER': 'db_ovim_user',
 185         'RO_DB_OVIM_PASSWORD': 'db_ovim_passwd',
 186         'RO_LOG_LEVEL': 'log_level',
 187         'RO_LOG_FILE': 'log_file',
 188     }
 189     ro_version = _get_version()
 190     # Configure logging step 1
 191     hostname = socket.gethostname()
 192     log_formatter_str = '%(asctime)s.%(msecs)03d00Z[{host}@openmanod] %(filename)s:%(lineno)s severity:%(levelname)s logger:%(name)s log:%(message)s'
 193     log_formatter_complete = logging.Formatter(log_formatter_str.format(host=hostname), datefmt='%Y-%m-%dT%H:%M:%S')
 194     log_format_simple = "%(asctime)s %(levelname)s  %(name)s %(thread)d %(filename)s:%(lineno)s %(message)s"
 195     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 196     logging.basicConfig(format=log_format_simple, level=logging.DEBUG)
 197     logger = logging.getLogger('openmano')
 198     logger.setLevel(logging.DEBUG)
 199     socket_handler = None
 200     # Read parameters and configuration file
 201     httpthread = None
 202     try:
 203         # load parameters and configuration
 204         opts, args = getopt.getopt(sys.argv[1:], "hvc:V:p:P:",
 205                                    ["config=", "help", "version", "port=", "vnf-repository=", "adminport=",
 206                                     "log-socket-host=", "log-socket-port=", "log-file=", "create-tenant="])
 207         port = None
 208         port_admin = None
 209         config_file = 'openmanod.cfg'
 210         vnf_repository = None
 211         log_file = None
 212         log_socket_host = None
 213         log_socket_port = None
 214         create_tenant = None
 215
 216         for o, a in opts:
 217             if o in ("-v", "--version"):
 218                 print ("openmanod version {} {}".format(ro_version, version_date))
 219                 print ("(c) Copyright Telefonica")
 220                 sys.exit()
 221             elif o in ("-h", "--help"):
 222                 usage()
 223                 sys.exit()
 224             elif o in ("-V", "--vnf-repository"):
 225                 vnf_repository = a
 226             elif o in ("-c", "--config"):
 227                 config_file = a
 228             elif o in ("-p", "--port"):
 229                 port = a
 230             elif o in ("-P", "--adminport"):
 231                 port_admin = a
 232             elif o == "--log-socket-port":
 233                 log_socket_port = a
 234             elif o == "--log-socket-host":
 235                 log_socket_host = a
 236             elif o == "--log-file":
 237                 log_file = a
 238             elif o == "--create-tenant":
 239                 create_tenant = a
 240             else:
 241                 assert False, "Unhandled option"
 242         if log_file:
 243             set_logging_file(log_file)
 244         global_config = load_configuration(config_file)
 245         global_config["version"] = ro_version
 246         global_config["version_date"] = version_date
 247         # Override parameters obtained by command line on ENV
 248         if port:
 249             global_config['http_port'] = port
 250         if port_admin:
 251             global_config['http_admin_port'] = port_admin
 252         if log_socket_host:
 253             global_config['log_socket_host'] = log_socket_host
 254         if log_socket_port:
 255             global_config['log_socket_port'] = log_socket_port
 256
 257         # override with ENV
 258         for env_k, env_v in environ.items():
 259             try:
 260                 if not env_k.startswith("RO_") or env_k not in env2config or not env_v:
 261                     continue
 262                 global_config[env2config[env_k]] = env_v
 263                 if env_k.endswith("PORT"):  # convert to int, skip if not possible
 264                     global_config[env2config[env_k]] = int(env_v)
 265             except Exception as e:
 266                 logger.warn("skipping environ '{}={}' because exception '{}'".format(env_k, env_v, e))
 267
 268         global_config["console_port_iterator"] = console_port_iterator
 269         global_config["console_thread"] = {}
 270         global_config["console_ports"] = {}
 271         if not global_config["http_console_host"]:
 272             global_config["http_console_host"] = global_config["http_host"]
 273             if global_config["http_host"] == "0.0.0.0":
 274                 global_config["http_console_host"] = socket.gethostname()
 275
 276         # Configure logging STEP 2
 277         if "log_host" in global_config:
 278             socket_handler = log_handlers.SocketHandler(global_config["log_socket_host"],
 279                                                         global_config["log_socket_port"])
 280             socket_handler.setFormatter(log_formatter_complete)
 281             if global_config.get("log_socket_level") \
 282                     and global_config["log_socket_level"] != global_config["log_level"]:
 283                 socket_handler.setLevel(global_config["log_socket_level"])
 284             logger.addHandler(socket_handler)
 285
 286         if log_file:
 287             global_config['log_file'] = log_file
 288         elif global_config.get('log_file'):
 289             set_logging_file(global_config['log_file'])
 290
 291         logger.setLevel(getattr(logging, global_config['log_level']))
 292         logger.critical("Starting openmano server version: '%s %s' command: '%s'",
 293                         ro_version, version_date, " ".join(sys.argv))
 294
 295         for log_module in ("nfvo", "http", "vim", "wim", "db", "console", "ovim", "sdn", "sdnconn"):
 296             log_level_module = "log_level_" + log_module
 297             log_file_module = "log_file_" + log_module
 298             logger_module = logging.getLogger('openmano.' + log_module)
 299             if log_level_module in global_config:
 300                 logger_module.setLevel(global_config[log_level_module])
 301             if log_file_module in global_config:
 302                 try:
 303                     file_handler = logging.handlers.RotatingFileHandler(global_config[log_file_module],
 304                                                                         maxBytes=100e6, backupCount=9, delay=0)
 305                     file_handler.setFormatter(log_formatter_simple)
 306                     logger_module.addHandler(file_handler)
 307                 except IOError as e:
 308                     raise LoadConfigurationException(
 309                         "Cannot open logging file '{}': {}. Check folder exist and permissions".format(
 310                             global_config[log_file_module], str(e)))
 311             global_config["logger_" + log_module] = logger_module
 312
 313         # Initialize DB connection
 314         mydb = nfvo_db.nfvo_db()
 315         mydb.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'],
 316                      global_config['db_name'])
 317         db_path = osm_ro.__path__[0] + "/database_utils"
 318         if not os_path.exists(db_path + "/migrate_mano_db.sh"):
 319             db_path = osm_ro.__path__[0] + "/../database_utils"
 320         try:
 321             r = mydb.get_db_version()
 322             if r[0] != database_version:
 323                 logger.critical("DATABASE wrong version '{current}'. Try to upgrade/downgrade to version '{target}'"
 324                                 " with '{db_path}/migrate_mano_db.sh {target}'".format(current=r[0],
 325                                                                                        target=database_version,
 326                                                                                        db_path=db_path))
 327                 exit(-1)
 328         except db_base_Exception as e:
 329             logger.critical("DATABASE is not valid. If you think it is corrupted, you can init it with"
 330                             " '{db_path}/init_mano_db.sh' script".format(db_path=db_path))
 331             exit(-1)
 332
 333         nfvo.global_config = global_config
 334         if create_tenant:
 335             try:
 336                 nfvo.new_tenant(mydb, {"name": create_tenant})
 337             except Exception as e:
 338                 if isinstance(e, nfvo.NfvoException) and e.http_code == 409:
 339                     pass  # if tenant exist (NfvoException error 409), ignore
 340                 else:  # otherwise print and error and continue
 341                     logger.error("Cannot create tenant '{}': {}".format(create_tenant, e))
 342
 343         # WIM module
 344         wim_persistence = WimPersistence(mydb)
 345         wim_engine = WimEngine(wim_persistence, nfvo.plugins)
 346         # ---
 347         nfvo.start_service(mydb, wim_persistence, wim_engine)
 348
 349         httpthread = httpserver.httpserver(
 350             mydb, False,
 351             global_config['http_host'], global_config['http_port'],
 352             wim_persistence, wim_engine
 353         )
 354
 355         httpthread.start()
 356         if 'http_admin_port' in global_config:
 357             httpthreadadmin = httpserver.httpserver(mydb, True, global_config['http_host'],
 358                                                     global_config['http_admin_port'])
 359             httpthreadadmin.start()
 360         time.sleep(1)
 361         logger.info('Waiting for http clients')
 362         print('Waiting for http clients')
 363         print('openmanod ready')
 364         print('====================')
 365         time.sleep(20)
 366         sys.stdout.flush()
 367
 368         # TODO: Interactive console must be implemented here instead of join or sleep
 369
 370         # httpthread.join()
 371         # if 'http_admin_port' in global_config:
 372         #    httpthreadadmin.join()
 373         while True:
 374             time.sleep(86400)
 375
 376     except KeyboardInterrupt as e:
 377         logger.info(str(e))
 378     except SystemExit:
 379         pass
 380     except getopt.GetoptError as e:
 381         logger.critical(str(e))  # will print something like "option -a not recognized"
 382         exit(-1)
 383     except LoadConfigurationException as e:
 384         logger.critical(str(e))
 385         exit(-1)
 386     except db_base_Exception as e:
 387         logger.critical(str(e))
 388         exit(-1)
 389     except nfvo.NfvoException as e:
 390         logger.critical(str(e), exc_info=True)
 391         exit(-1)
 392     nfvo.stop_service()
 393     if httpthread:
 394         httpthread.join(1)