NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 import cherrypy
  22 import time
  23 import json
  24 import yaml
  25 import osm_ng_ro.html_out as html
  26 import logging
  27 import logging.handlers
  28 import getopt
  29 import sys
  30
  31 from osm_ng_ro.ns import Ns, NsException
  32 from osm_ng_ro.validation import ValidationError
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from http import HTTPStatus
  37 from codecs import getreader
  38 from os import environ, path
  39 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  40
  41 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  42
  43 __version__ = "0.1."    # file version, not NBI version
  44 version_date = "May 2020"
  45
  46 database_version = '1.2'
  47 auth_database_version = '1.0'
  48 ro_server = None           # instance of Server class
  49 # vim_threads = None  # instance of VimThread class
  50
  51 """
  52 RO North Bound Interface
  53 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  54         /ns/v1/deploy                                           O
  55             /<nsrs_id>                                          O       O               O
  56                 /<action_id>                                    O
  57                     /cancel                                             O
  58
  59 """
  60
  61 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  62 # ^ Contains possible administrative query string words:
  63 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  64 #           (not owned by my session project).
  65 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  66 #     FORCE=True(by default)|False: Force edition/deletion operations
  67 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  68
  69 valid_url_methods = {
  70     # contains allowed URL and methods, and the role_permission name
  71     "admin": {
  72         "v1": {
  73             "tokens": {
  74                 "METHODS": ("POST",),
  75                 "ROLE_PERMISSION": "tokens:",
  76                 "<ID>": {
  77                     "METHODS": ("DELETE",),
  78                     "ROLE_PERMISSION": "tokens:id:"
  79                 }
  80             },
  81         }
  82     },
  83     "ns": {
  84         "v1": {
  85             "deploy": {
  86                 "METHODS": ("GET",),
  87                 "ROLE_PERMISSION": "deploy:",
  88                 "<ID>": {
  89                     "METHODS": ("GET", "POST", "DELETE"),
  90                     "ROLE_PERMISSION": "deploy:id:",
  91                     "<ID>": {
  92                         "METHODS": ("GET",),
  93                         "ROLE_PERMISSION": "deploy:id:id:",
  94                         "cancel": {
  95                             "METHODS": ("POST",),
  96                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  97                         }
  98                     }
  99                 }
 100             },
 101         }
 102     },
 103 }
 104
 105
 106 class RoException(Exception):
 107
 108     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 109         Exception.__init__(self, message)
 110         self.http_code = http_code
 111
 112
 113 class AuthException(RoException):
 114     pass
 115
 116
 117 class Authenticator:
 118
 119     def __init__(self, valid_url_methods, valid_query_string):
 120         self.valid_url_methods = valid_url_methods
 121         self.valid_query_string = valid_query_string
 122
 123     def authorize(self, *args, **kwargs):
 124         return {"token": "ok", "id": "ok"}
 125
 126     def new_token(self, token_info, indata, remote):
 127         return {"token": "ok",
 128                 "id": "ok",
 129                 "remote": remote}
 130
 131     def del_token(self, token_id):
 132         pass
 133
 134     def start(self, engine_config):
 135         pass
 136
 137
 138 class Server(object):
 139     instance = 0
 140     # to decode bytes to str
 141     reader = getreader("utf-8")
 142
 143     def __init__(self):
 144         self.instance += 1
 145         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 146         self.ns = Ns()
 147         self.map_operation = {
 148             "token:post": self.new_token,
 149             "token:id:delete": self.del_token,
 150             "deploy:get": self.ns.get_deploy,
 151             "deploy:id:get": self.ns.get_actions,
 152             "deploy:id:post": self.ns.deploy,
 153             "deploy:id:delete": self.ns.delete,
 154             "deploy:id:id:get": self.ns.status,
 155             "deploy:id:id:cancel:post": self.ns.cancel,
 156         }
 157
 158     def _format_in(self, kwargs):
 159         try:
 160             indata = None
 161             if cherrypy.request.body.length:
 162                 error_text = "Invalid input format "
 163
 164                 if "Content-Type" in cherrypy.request.headers:
 165                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 166                         error_text = "Invalid json format "
 167                         indata = json.load(self.reader(cherrypy.request.body))
 168                         cherrypy.request.headers.pop("Content-File-MD5", None)
 169                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 170                         error_text = "Invalid yaml format "
 171                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 172                         cherrypy.request.headers.pop("Content-File-MD5", None)
 173                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 174                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 175                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 176                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 177                         indata = cherrypy.request.body  # .read()
 178                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 179                         if "descriptor_file" in kwargs:
 180                             filecontent = kwargs.pop("descriptor_file")
 181                             if not filecontent.file:
 182                                 raise RoException("empty file or content", HTTPStatus.BAD_REQUEST)
 183                             indata = filecontent.file  # .read()
 184                             if filecontent.content_type.value:
 185                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 186                     else:
 187                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 188                         #                          "Only 'Content-Type' of type 'application/json' or
 189                         # 'application/yaml' for input format are available")
 190                         error_text = "Invalid yaml format "
 191                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 192                         cherrypy.request.headers.pop("Content-File-MD5", None)
 193                 else:
 194                     error_text = "Invalid yaml format "
 195                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 196                     cherrypy.request.headers.pop("Content-File-MD5", None)
 197             if not indata:
 198                 indata = {}
 199
 200             format_yaml = False
 201             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 202                 format_yaml = True
 203
 204             for k, v in kwargs.items():
 205                 if isinstance(v, str):
 206                     if v == "":
 207                         kwargs[k] = None
 208                     elif format_yaml:
 209                         try:
 210                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 211                         except Exception:
 212                             pass
 213                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 214                         try:
 215                             kwargs[k] = int(v)
 216                         except Exception:
 217                             try:
 218                                 kwargs[k] = float(v)
 219                             except Exception:
 220                                 pass
 221                     elif v.find(",") > 0:
 222                         kwargs[k] = v.split(",")
 223                 elif isinstance(v, (list, tuple)):
 224                     for index in range(0, len(v)):
 225                         if v[index] == "":
 226                             v[index] = None
 227                         elif format_yaml:
 228                             try:
 229                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 230                             except Exception:
 231                                 pass
 232
 233             return indata
 234         except (ValueError, yaml.YAMLError) as exc:
 235             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 236         except KeyError as exc:
 237             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 238         except Exception as exc:
 239             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 240
 241     @staticmethod
 242     def _format_out(data, token_info=None, _format=None):
 243         """
 244         return string of dictionary data according to requested json, yaml, xml. By default json
 245         :param data: response to be sent. Can be a dict, text or file
 246         :param token_info: Contains among other username and project
 247         :param _format: The format to be set as Content-Type if data is a file
 248         :return: None
 249         """
 250         accept = cherrypy.request.headers.get("Accept")
 251         if data is None:
 252             if accept and "text/html" in accept:
 253                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 254             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 255             return
 256         elif hasattr(data, "read"):  # file object
 257             if _format:
 258                 cherrypy.response.headers["Content-Type"] = _format
 259             elif "b" in data.mode:  # binariy asssumig zip
 260                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 261             else:
 262                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 263             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 264             return data
 265         if accept:
 266             if "application/json" in accept:
 267                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 268                 a = json.dumps(data, indent=4) + "\n"
 269                 return a.encode("utf8")
 270             elif "text/html" in accept:
 271                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 272
 273             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 274                 pass
 275             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 276             elif cherrypy.response.status >= 400:
 277                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 278                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 279                                          "for output format are available")
 280         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 281         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 282                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 283
 284     @cherrypy.expose
 285     def index(self, *args, **kwargs):
 286         token_info = None
 287         try:
 288             if cherrypy.request.method == "GET":
 289                 token_info = self.authenticator.authorize()
 290                 outdata = token_info   # Home page
 291             else:
 292                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 293                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 294
 295             return self._format_out(outdata, token_info)
 296
 297         except (NsException, AuthException) as e:
 298             # cherrypy.log("index Exception {}".format(e))
 299             cherrypy.response.status = e.http_code.value
 300             return self._format_out("Welcome to OSM!", token_info)
 301
 302     @cherrypy.expose
 303     def version(self, *args, **kwargs):
 304         # TODO consider to remove and provide version using the static version file
 305         try:
 306             if cherrypy.request.method != "GET":
 307                 raise RoException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 308             elif args or kwargs:
 309                 raise RoException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 310             # TODO include version of other modules, pick up from some kafka admin message
 311             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 312             return self._format_out(osm_ng_ro_version)
 313         except RoException as e:
 314             cherrypy.response.status = e.http_code.value
 315             problem_details = {
 316                 "code": e.http_code.name,
 317                 "status": e.http_code.value,
 318                 "detail": str(e),
 319             }
 320             return self._format_out(problem_details, None)
 321
 322     def new_token(self, engine_session, indata, *args, **kwargs):
 323         token_info = None
 324
 325         try:
 326             token_info = self.authenticator.authorize()
 327         except Exception:
 328             token_info = None
 329         if kwargs:
 330             indata.update(kwargs)
 331         # This is needed to log the user when authentication fails
 332         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 333         token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 334         cherrypy.session['Authorization'] = token_info["id"]
 335         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 336         # for logging
 337
 338         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 339         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 340         return token_info, token_info["id"], True
 341
 342     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 343         token_id = _id
 344         if not token_id and "id" in kwargs:
 345             token_id = kwargs["id"]
 346         elif not token_id:
 347             token_info = self.authenticator.authorize()
 348             # for logging
 349             token_id = token_info["id"]
 350         self.authenticator.del_token(token_id)
 351         token_info = None
 352         cherrypy.session['Authorization'] = "logout"
 353         # cherrypy.response.cookie["Authorization"] = token_id
 354         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 355         return None, None, True
 356
 357     @cherrypy.expose
 358     def test(self, *args, **kwargs):
 359         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 360                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 361             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 362             return "test URL is disabled"
 363         thread_info = None
 364         if args and args[0] == "help":
 365             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 366                    "sleep/<time>\nmessage/topic\n</pre></html>"
 367
 368         elif args and args[0] == "init":
 369             try:
 370                 # self.ns.load_dbase(cherrypy.request.app.config)
 371                 self.ns.create_admin()
 372                 return "Done. User 'admin', password 'admin' created"
 373             except Exception:
 374                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 375                 return self._format_out("Database already initialized")
 376         elif args and args[0] == "file":
 377             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/ro'].config["storage"]["path"] + "/" + args[1],
 378                                                   "text/plain", "attachment")
 379         elif args and args[0] == "file2":
 380             f_path = cherrypy.tree.apps['/ro'].config["storage"]["path"] + "/" + args[1]
 381             f = open(f_path, "r")
 382             cherrypy.response.headers["Content-type"] = "text/plain"
 383             return f
 384
 385         elif len(args) == 2 and args[0] == "db-clear":
 386             deleted_info = self.ns.db.del_list(args[1], kwargs)
 387             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 388         elif len(args) and args[0] == "fs-clear":
 389             if len(args) >= 2:
 390                 folders = (args[1],)
 391             else:
 392                 folders = self.ns.fs.dir_ls(".")
 393             for folder in folders:
 394                 self.ns.fs.file_delete(folder)
 395             return ",".join(folders) + " folders deleted\n"
 396         elif args and args[0] == "login":
 397             if not cherrypy.request.headers.get("Authorization"):
 398                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 399                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 400         elif args and args[0] == "login2":
 401             if not cherrypy.request.headers.get("Authorization"):
 402                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 403                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 404         elif args and args[0] == "sleep":
 405             sleep_time = 5
 406             try:
 407                 sleep_time = int(args[1])
 408             except Exception:
 409                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 410                 return self._format_out("Database already initialized")
 411             thread_info = cherrypy.thread_data
 412             print(thread_info)
 413             time.sleep(sleep_time)
 414             # thread_info
 415         elif len(args) >= 2 and args[0] == "message":
 416             main_topic = args[1]
 417             return_text = "<html><pre>{} ->\n".format(main_topic)
 418             try:
 419                 if cherrypy.request.method == 'POST':
 420                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 421                     for k, v in to_send.items():
 422                         self.ns.msg.write(main_topic, k, v)
 423                         return_text += "  {}: {}\n".format(k, v)
 424                 elif cherrypy.request.method == 'GET':
 425                     for k, v in kwargs.items():
 426                         self.ns.msg.write(main_topic, k, yaml.load(v, Loader=yaml.SafeLoader))
 427                         return_text += "  {}: {}\n".format(k, yaml.load(v, Loader=yaml.SafeLoader))
 428             except Exception as e:
 429                 return_text += "Error: " + str(e)
 430             return_text += "</pre></html>\n"
 431             return return_text
 432
 433         return_text = (
 434             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 435             "  kwargs: {}\n".format(kwargs) +
 436             "  headers: {}\n".format(cherrypy.request.headers) +
 437             "  path_info: {}\n".format(cherrypy.request.path_info) +
 438             "  query_string: {}\n".format(cherrypy.request.query_string) +
 439             "  session: {}\n".format(cherrypy.session) +
 440             "  cookie: {}\n".format(cherrypy.request.cookie) +
 441             "  method: {}\n".format(cherrypy.request.method) +
 442             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 443             "  body:\n")
 444         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 445         if cherrypy.request.body.length:
 446             return_text += "    content: {}\n".format(
 447                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 448         if thread_info:
 449             return_text += "thread: {}\n".format(thread_info)
 450         return_text += "</pre></html>"
 451         return return_text
 452
 453     @staticmethod
 454     def _check_valid_url_method(method, *args):
 455         if len(args) < 3:
 456             raise RoException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 457
 458         reference = valid_url_methods
 459         for arg in args:
 460             if arg is None:
 461                 break
 462             if not isinstance(reference, dict):
 463                 raise RoException("URL contains unexpected extra items '{}'".format(arg),
 464                                   HTTPStatus.METHOD_NOT_ALLOWED)
 465
 466             if arg in reference:
 467                 reference = reference[arg]
 468             elif "<ID>" in reference:
 469                 reference = reference["<ID>"]
 470             elif "*" in reference:
 471                 # reference = reference["*"]
 472                 break
 473             else:
 474                 raise RoException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 475         if "TODO" in reference and method in reference["TODO"]:
 476             raise RoException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 477         elif "METHODS" not in reference or method not in reference["METHODS"]:
 478             raise RoException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 479         return reference["ROLE_PERMISSION"] + method.lower()
 480
 481     @staticmethod
 482     def _set_location_header(main_topic, version, topic, id):
 483         """
 484         Insert response header Location with the URL of created item base on URL params
 485         :param main_topic:
 486         :param version:
 487         :param topic:
 488         :param id:
 489         :return: None
 490         """
 491         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 492         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(main_topic, version, topic, id)
 493         return
 494
 495     @cherrypy.expose
 496     def default(self, main_topic=None, version=None, topic=None, _id=None, _id2=None, *args, **kwargs):
 497         token_info = None
 498         outdata = None
 499         _format = None
 500         method = "DONE"
 501         rollback = []
 502         engine_session = None
 503         try:
 504             if not main_topic or not version or not topic:
 505                 raise RoException("URL must contain at least 'main_topic/version/topic'",
 506                                   HTTPStatus.METHOD_NOT_ALLOWED)
 507             if main_topic not in ("admin", "ns",):
 508                 raise RoException("URL main_topic '{}' not supported".format(main_topic),
 509                                   HTTPStatus.METHOD_NOT_ALLOWED)
 510             if version != 'v1':
 511                 raise RoException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 512
 513             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 514                 method = kwargs.pop("METHOD")
 515             else:
 516                 method = cherrypy.request.method
 517
 518             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, _id2, *args,
 519                                                            **kwargs)
 520             # skip token validation if requesting a token
 521             indata = self._format_in(kwargs)
 522             if main_topic != "admin" or topic != "tokens":
 523                 token_info = self.authenticator.authorize(role_permission, _id)
 524             outdata, created_id, done = self.map_operation[role_permission](
 525                 engine_session, indata, version, _id, _id2, *args, *kwargs)
 526             if created_id:
 527                 self._set_location_header(main_topic, version, topic, _id)
 528             cherrypy.response.status = HTTPStatus.ACCEPTED.value if not done else HTTPStatus.OK.value if \
 529                 outdata is not None else HTTPStatus.NO_CONTENT.value
 530             return self._format_out(outdata, token_info, _format)
 531         except Exception as e:
 532             if isinstance(e, (RoException, NsException, DbException, FsException, MsgException, AuthException,
 533                               ValidationError)):
 534                 http_code_value = cherrypy.response.status = e.http_code.value
 535                 http_code_name = e.http_code.name
 536                 cherrypy.log("Exception {}".format(e))
 537             else:
 538                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 539                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 540                 http_code_name = HTTPStatus.BAD_REQUEST.name
 541             if hasattr(outdata, "close"):  # is an open file
 542                 outdata.close()
 543             error_text = str(e)
 544             rollback.reverse()
 545             for rollback_item in rollback:
 546                 try:
 547                     if rollback_item.get("operation") == "set":
 548                         self.ns.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
 549                                            rollback_item["content"], fail_on_empty=False)
 550                     else:
 551                         self.ns.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
 552                                            fail_on_empty=False)
 553                 except Exception as e2:
 554                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
 555                     cherrypy.log(rollback_error_text)
 556                     error_text += ". " + rollback_error_text
 557             # if isinstance(e, MsgException):
 558             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 559             #         engine_topic[:-1], method, error_text)
 560             problem_details = {
 561                 "code": http_code_name,
 562                 "status": http_code_value,
 563                 "detail": error_text,
 564             }
 565             return self._format_out(problem_details, token_info)
 566             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 567         finally:
 568             if token_info:
 569                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 570                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 571                         if outdata.get(logging_id):
 572                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
 573
 574
 575 def _start_service():
 576     """
 577     Callback function called when cherrypy.engine starts
 578     Override configuration with env variables
 579     Set database, storage, message configuration
 580     Init database with admin/admin user password
 581     """
 582     global ro_server
 583     # global vim_threads
 584     cherrypy.log.error("Starting osm_ng_ro")
 585     # update general cherrypy configuration
 586     update_dict = {}
 587
 588     engine_config = cherrypy.tree.apps['/ro'].config
 589     for k, v in environ.items():
 590         if not k.startswith("OSMRO_"):
 591             continue
 592         k1, _, k2 = k[6:].lower().partition("_")
 593         if not k2:
 594             continue
 595         try:
 596             if k1 in ("server", "test", "auth", "log"):
 597                 # update [global] configuration
 598                 update_dict[k1 + '.' + k2] = yaml.safe_load(v)
 599             elif k1 == "static":
 600                 # update [/static] configuration
 601                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 602             elif k1 == "tools":
 603                 # update [/] configuration
 604                 engine_config["/"]["tools." + k2.replace('_', '.')] = yaml.safe_load(v)
 605             elif k1 in ("message", "database", "storage", "authentication"):
 606                 # update [message], [database], ... configuration
 607                 if k2 in ("port", "db_port"):
 608                     engine_config[k1][k2] = int(v)
 609                 else:
 610                     engine_config[k1][k2] = v
 611
 612         except Exception as e:
 613             raise RoException("Cannot load env '{}': {}".format(k, e))
 614
 615     if update_dict:
 616         cherrypy.config.update(update_dict)
 617         engine_config["global"].update(update_dict)
 618
 619     # logging cherrypy
 620     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 621     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 622     logger_server = logging.getLogger("cherrypy.error")
 623     logger_access = logging.getLogger("cherrypy.access")
 624     logger_cherry = logging.getLogger("cherrypy")
 625     logger_nbi = logging.getLogger("ro")
 626
 627     if "log.file" in engine_config["global"]:
 628         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
 629                                                             maxBytes=100e6, backupCount=9, delay=0)
 630         file_handler.setFormatter(log_formatter_simple)
 631         logger_cherry.addHandler(file_handler)
 632         logger_nbi.addHandler(file_handler)
 633     # log always to standard output
 634     for format_, logger in {"ro.server %(filename)s:%(lineno)s": logger_server,
 635                             "ro.access %(filename)s:%(lineno)s": logger_access,
 636                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
 637                             }.items():
 638         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 639         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
 640         str_handler = logging.StreamHandler()
 641         str_handler.setFormatter(log_formatter_cherry)
 642         logger.addHandler(str_handler)
 643
 644     if engine_config["global"].get("log.level"):
 645         logger_cherry.setLevel(engine_config["global"]["log.level"])
 646         logger_nbi.setLevel(engine_config["global"]["log.level"])
 647
 648     # logging other modules
 649     for k1, logname in {"message": "ro.msg", "database": "ro.db", "storage": "ro.fs"}.items():
 650         engine_config[k1]["logger_name"] = logname
 651         logger_module = logging.getLogger(logname)
 652         if "logfile" in engine_config[k1]:
 653             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
 654                                                                 maxBytes=100e6, backupCount=9, delay=0)
 655             file_handler.setFormatter(log_formatter_simple)
 656             logger_module.addHandler(file_handler)
 657         if "loglevel" in engine_config[k1]:
 658             logger_module.setLevel(engine_config[k1]["loglevel"])
 659     # TODO add more entries, e.g.: storage
 660
 661     engine_config["assignment"] = {}
 662     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 663     cherrypy.tree.apps['/ro'].root.ns.start(engine_config)
 664     cherrypy.tree.apps['/ro'].root.authenticator.start(engine_config)
 665     cherrypy.tree.apps['/ro'].root.ns.init_db(target_version=database_version)
 666
 667     # # start subscriptions thread:
 668     # vim_threads = []
 669     # for thread_id in range(engine_config["global"]["server.ns_threads"]):
 670     #     vim_thread = VimThread(thread_id, config=engine_config, engine=ro_server.ns)
 671     #     vim_thread.start()
 672     #     vim_threads.append(vim_thread)
 673     # # Do not capture except SubscriptionException
 674
 675     backend = engine_config["authentication"]["backend"]
 676     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 677                        .format(ro_version, ro_version_date, backend))
 678
 679
 680 def _stop_service():
 681     """
 682     Callback function called when cherrypy.engine stops
 683     TODO: Ending database connections.
 684     """
 685     # global vim_threads
 686     # if vim_threads:
 687     #     for vim_thread in vim_threads:
 688     #         vim_thread.terminate()
 689     # vim_threads = None
 690     cherrypy.tree.apps['/ro'].root.ns.stop()
 691     cherrypy.log.error("Stopping osm_ng_ro")
 692
 693
 694 def ro_main(config_file):
 695     global ro_server
 696     ro_server = Server()
 697     cherrypy.engine.subscribe('start', _start_service)
 698     cherrypy.engine.subscribe('stop', _stop_service)
 699     cherrypy.quickstart(ro_server, '/ro', config_file)
 700
 701
 702 def usage():
 703     print("""Usage: {} [options]
 704         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 705         -h|--help: shows this help
 706         """.format(sys.argv[0]))
 707     # --log-socket-host HOST: send logs to this host")
 708     # --log-socket-port PORT: send logs using this port (default: 9022)")
 709
 710
 711 if __name__ == '__main__':
 712     try:
 713         # load parameters and configuration
 714         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 715         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 716         config_file = None
 717         for o, a in opts:
 718             if o in ("-h", "--help"):
 719                 usage()
 720                 sys.exit()
 721             elif o in ("-c", "--config"):
 722                 config_file = a
 723             else:
 724                 assert False, "Unhandled option"
 725         if config_file:
 726             if not path.isfile(config_file):
 727                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
 728                 exit(1)
 729         else:
 730             for config_file in (path.dirname(__file__) + "/ro.cfg", "./ro.cfg", "/etc/osm/ro.cfg"):
 731                 if path.isfile(config_file):
 732                     break
 733             else:
 734                 print("No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
 735                 exit(1)
 736         ro_main(config_file)
 737     except getopt.GetoptError as e:
 738         print(str(e), file=sys.stderr)
 739         # usage()
 740         exit(1)