NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 import cherrypy
  22 import time
  23 import json
  24 import yaml
  25 import osm_ng_ro.html_out as html
  26 import logging
  27 import logging.handlers
  28 import getopt
  29 import sys
  30
  31 from osm_ng_ro.ns import Ns, NsException
  32 from osm_ng_ro.validation import ValidationError
  33 from osm_ng_ro.vim_admin import VimAdminThread
  34 from osm_common.dbbase import DbException
  35 from osm_common.fsbase import FsException
  36 from osm_common.msgbase import MsgException
  37 from http import HTTPStatus
  38 from codecs import getreader
  39 from os import environ, path
  40 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  41
  42 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  43
  44 __version__ = "0.1."    # file version, not NBI version
  45 version_date = "May 2020"
  46
  47 database_version = '1.2'
  48 auth_database_version = '1.0'
  49 ro_server = None           # instance of Server class
  50 vim_admin_thread = None   # instance of VimAdminThread class
  51
  52 # vim_threads = None  # instance of VimThread class
  53
  54 """
  55 RO North Bound Interface
  56 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  57         /ns/v1/deploy                                           O
  58             /<nsrs_id>                                          O       O               O
  59                 /<action_id>                                    O
  60                     /cancel                                             O
  61
  62 """
  63
  64 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  65 # ^ Contains possible administrative query string words:
  66 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  67 #           (not owned by my session project).
  68 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  69 #     FORCE=True(by default)|False: Force edition/deletion operations
  70 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  71
  72 valid_url_methods = {
  73     # contains allowed URL and methods, and the role_permission name
  74     "admin": {
  75         "v1": {
  76             "tokens": {
  77                 "METHODS": ("POST",),
  78                 "ROLE_PERMISSION": "tokens:",
  79                 "<ID>": {
  80                     "METHODS": ("DELETE",),
  81                     "ROLE_PERMISSION": "tokens:id:"
  82                 }
  83             },
  84         }
  85     },
  86     "ns": {
  87         "v1": {
  88             "deploy": {
  89                 "METHODS": ("GET",),
  90                 "ROLE_PERMISSION": "deploy:",
  91                 "<ID>": {
  92                     "METHODS": ("GET", "POST", "DELETE"),
  93                     "ROLE_PERMISSION": "deploy:id:",
  94                     "<ID>": {
  95                         "METHODS": ("GET",),
  96                         "ROLE_PERMISSION": "deploy:id:id:",
  97                         "cancel": {
  98                             "METHODS": ("POST",),
  99                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 100                         }
 101                     }
 102                 }
 103             },
 104         }
 105     },
 106 }
 107
 108
 109 class RoException(Exception):
 110
 111     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 112         Exception.__init__(self, message)
 113         self.http_code = http_code
 114
 115
 116 class AuthException(RoException):
 117     pass
 118
 119
 120 class Authenticator:
 121
 122     def __init__(self, valid_url_methods, valid_query_string):
 123         self.valid_url_methods = valid_url_methods
 124         self.valid_query_string = valid_query_string
 125
 126     def authorize(self, *args, **kwargs):
 127         return {"token": "ok", "id": "ok"}
 128
 129     def new_token(self, token_info, indata, remote):
 130         return {"token": "ok",
 131                 "id": "ok",
 132                 "remote": remote}
 133
 134     def del_token(self, token_id):
 135         pass
 136
 137     def start(self, engine_config):
 138         pass
 139
 140
 141 class Server(object):
 142     instance = 0
 143     # to decode bytes to str
 144     reader = getreader("utf-8")
 145
 146     def __init__(self):
 147         self.instance += 1
 148         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 149         self.ns = Ns()
 150         self.map_operation = {
 151             "token:post": self.new_token,
 152             "token:id:delete": self.del_token,
 153             "deploy:get": self.ns.get_deploy,
 154             "deploy:id:get": self.ns.get_actions,
 155             "deploy:id:post": self.ns.deploy,
 156             "deploy:id:delete": self.ns.delete,
 157             "deploy:id:id:get": self.ns.status,
 158             "deploy:id:id:cancel:post": self.ns.cancel,
 159         }
 160
 161     def _format_in(self, kwargs):
 162         try:
 163             indata = None
 164             if cherrypy.request.body.length:
 165                 error_text = "Invalid input format "
 166
 167                 if "Content-Type" in cherrypy.request.headers:
 168                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 169                         error_text = "Invalid json format "
 170                         indata = json.load(self.reader(cherrypy.request.body))
 171                         cherrypy.request.headers.pop("Content-File-MD5", None)
 172                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 173                         error_text = "Invalid yaml format "
 174                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 175                         cherrypy.request.headers.pop("Content-File-MD5", None)
 176                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 177                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 178                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 179                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 180                         indata = cherrypy.request.body  # .read()
 181                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 182                         if "descriptor_file" in kwargs:
 183                             filecontent = kwargs.pop("descriptor_file")
 184                             if not filecontent.file:
 185                                 raise RoException("empty file or content", HTTPStatus.BAD_REQUEST)
 186                             indata = filecontent.file  # .read()
 187                             if filecontent.content_type.value:
 188                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 189                     else:
 190                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 191                         #                          "Only 'Content-Type' of type 'application/json' or
 192                         # 'application/yaml' for input format are available")
 193                         error_text = "Invalid yaml format "
 194                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 195                         cherrypy.request.headers.pop("Content-File-MD5", None)
 196                 else:
 197                     error_text = "Invalid yaml format "
 198                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 199                     cherrypy.request.headers.pop("Content-File-MD5", None)
 200             if not indata:
 201                 indata = {}
 202
 203             format_yaml = False
 204             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 205                 format_yaml = True
 206
 207             for k, v in kwargs.items():
 208                 if isinstance(v, str):
 209                     if v == "":
 210                         kwargs[k] = None
 211                     elif format_yaml:
 212                         try:
 213                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 214                         except Exception:
 215                             pass
 216                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 217                         try:
 218                             kwargs[k] = int(v)
 219                         except Exception:
 220                             try:
 221                                 kwargs[k] = float(v)
 222                             except Exception:
 223                                 pass
 224                     elif v.find(",") > 0:
 225                         kwargs[k] = v.split(",")
 226                 elif isinstance(v, (list, tuple)):
 227                     for index in range(0, len(v)):
 228                         if v[index] == "":
 229                             v[index] = None
 230                         elif format_yaml:
 231                             try:
 232                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 233                             except Exception:
 234                                 pass
 235
 236             return indata
 237         except (ValueError, yaml.YAMLError) as exc:
 238             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 239         except KeyError as exc:
 240             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 241         except Exception as exc:
 242             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 243
 244     @staticmethod
 245     def _format_out(data, token_info=None, _format=None):
 246         """
 247         return string of dictionary data according to requested json, yaml, xml. By default json
 248         :param data: response to be sent. Can be a dict, text or file
 249         :param token_info: Contains among other username and project
 250         :param _format: The format to be set as Content-Type if data is a file
 251         :return: None
 252         """
 253         accept = cherrypy.request.headers.get("Accept")
 254         if data is None:
 255             if accept and "text/html" in accept:
 256                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 257             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 258             return
 259         elif hasattr(data, "read"):  # file object
 260             if _format:
 261                 cherrypy.response.headers["Content-Type"] = _format
 262             elif "b" in data.mode:  # binariy asssumig zip
 263                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 264             else:
 265                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 266             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 267             return data
 268         if accept:
 269             if "application/json" in accept:
 270                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 271                 a = json.dumps(data, indent=4) + "\n"
 272                 return a.encode("utf8")
 273             elif "text/html" in accept:
 274                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 275
 276             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 277                 pass
 278             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 279             elif cherrypy.response.status >= 400:
 280                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 281                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 282                                          "for output format are available")
 283         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 284         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 285                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 286
 287     @cherrypy.expose
 288     def index(self, *args, **kwargs):
 289         token_info = None
 290         try:
 291             if cherrypy.request.method == "GET":
 292                 token_info = self.authenticator.authorize()
 293                 outdata = token_info   # Home page
 294             else:
 295                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 296                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 297
 298             return self._format_out(outdata, token_info)
 299
 300         except (NsException, AuthException) as e:
 301             # cherrypy.log("index Exception {}".format(e))
 302             cherrypy.response.status = e.http_code.value
 303             return self._format_out("Welcome to OSM!", token_info)
 304
 305     @cherrypy.expose
 306     def version(self, *args, **kwargs):
 307         # TODO consider to remove and provide version using the static version file
 308         try:
 309             if cherrypy.request.method != "GET":
 310                 raise RoException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 311             elif args or kwargs:
 312                 raise RoException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 313             # TODO include version of other modules, pick up from some kafka admin message
 314             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 315             return self._format_out(osm_ng_ro_version)
 316         except RoException as e:
 317             cherrypy.response.status = e.http_code.value
 318             problem_details = {
 319                 "code": e.http_code.name,
 320                 "status": e.http_code.value,
 321                 "detail": str(e),
 322             }
 323             return self._format_out(problem_details, None)
 324
 325     def new_token(self, engine_session, indata, *args, **kwargs):
 326         token_info = None
 327
 328         try:
 329             token_info = self.authenticator.authorize()
 330         except Exception:
 331             token_info = None
 332         if kwargs:
 333             indata.update(kwargs)
 334         # This is needed to log the user when authentication fails
 335         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 336         token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 337         cherrypy.session['Authorization'] = token_info["id"]
 338         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 339         # for logging
 340
 341         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 342         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 343         return token_info, token_info["id"], True
 344
 345     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 346         token_id = _id
 347         if not token_id and "id" in kwargs:
 348             token_id = kwargs["id"]
 349         elif not token_id:
 350             token_info = self.authenticator.authorize()
 351             # for logging
 352             token_id = token_info["id"]
 353         self.authenticator.del_token(token_id)
 354         token_info = None
 355         cherrypy.session['Authorization'] = "logout"
 356         # cherrypy.response.cookie["Authorization"] = token_id
 357         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 358         return None, None, True
 359
 360     @cherrypy.expose
 361     def test(self, *args, **kwargs):
 362         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 363                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 364             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 365             return "test URL is disabled"
 366         thread_info = None
 367         if args and args[0] == "help":
 368             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 369                    "sleep/<time>\nmessage/topic\n</pre></html>"
 370
 371         elif args and args[0] == "init":
 372             try:
 373                 # self.ns.load_dbase(cherrypy.request.app.config)
 374                 self.ns.create_admin()
 375                 return "Done. User 'admin', password 'admin' created"
 376             except Exception:
 377                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 378                 return self._format_out("Database already initialized")
 379         elif args and args[0] == "file":
 380             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/ro'].config["storage"]["path"] + "/" + args[1],
 381                                                   "text/plain", "attachment")
 382         elif args and args[0] == "file2":
 383             f_path = cherrypy.tree.apps['/ro'].config["storage"]["path"] + "/" + args[1]
 384             f = open(f_path, "r")
 385             cherrypy.response.headers["Content-type"] = "text/plain"
 386             return f
 387
 388         elif len(args) == 2 and args[0] == "db-clear":
 389             deleted_info = self.ns.db.del_list(args[1], kwargs)
 390             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 391         elif len(args) and args[0] == "fs-clear":
 392             if len(args) >= 2:
 393                 folders = (args[1],)
 394             else:
 395                 folders = self.ns.fs.dir_ls(".")
 396             for folder in folders:
 397                 self.ns.fs.file_delete(folder)
 398             return ",".join(folders) + " folders deleted\n"
 399         elif args and args[0] == "login":
 400             if not cherrypy.request.headers.get("Authorization"):
 401                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 402                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 403         elif args and args[0] == "login2":
 404             if not cherrypy.request.headers.get("Authorization"):
 405                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 406                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 407         elif args and args[0] == "sleep":
 408             sleep_time = 5
 409             try:
 410                 sleep_time = int(args[1])
 411             except Exception:
 412                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 413                 return self._format_out("Database already initialized")
 414             thread_info = cherrypy.thread_data
 415             print(thread_info)
 416             time.sleep(sleep_time)
 417             # thread_info
 418         elif len(args) >= 2 and args[0] == "message":
 419             main_topic = args[1]
 420             return_text = "<html><pre>{} ->\n".format(main_topic)
 421             try:
 422                 if cherrypy.request.method == 'POST':
 423                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 424                     for k, v in to_send.items():
 425                         self.ns.msg.write(main_topic, k, v)
 426                         return_text += "  {}: {}\n".format(k, v)
 427                 elif cherrypy.request.method == 'GET':
 428                     for k, v in kwargs.items():
 429                         self.ns.msg.write(main_topic, k, yaml.load(v, Loader=yaml.SafeLoader))
 430                         return_text += "  {}: {}\n".format(k, yaml.load(v, Loader=yaml.SafeLoader))
 431             except Exception as e:
 432                 return_text += "Error: " + str(e)
 433             return_text += "</pre></html>\n"
 434             return return_text
 435
 436         return_text = (
 437             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 438             "  kwargs: {}\n".format(kwargs) +
 439             "  headers: {}\n".format(cherrypy.request.headers) +
 440             "  path_info: {}\n".format(cherrypy.request.path_info) +
 441             "  query_string: {}\n".format(cherrypy.request.query_string) +
 442             "  session: {}\n".format(cherrypy.session) +
 443             "  cookie: {}\n".format(cherrypy.request.cookie) +
 444             "  method: {}\n".format(cherrypy.request.method) +
 445             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 446             "  body:\n")
 447         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 448         if cherrypy.request.body.length:
 449             return_text += "    content: {}\n".format(
 450                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 451         if thread_info:
 452             return_text += "thread: {}\n".format(thread_info)
 453         return_text += "</pre></html>"
 454         return return_text
 455
 456     @staticmethod
 457     def _check_valid_url_method(method, *args):
 458         if len(args) < 3:
 459             raise RoException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 460
 461         reference = valid_url_methods
 462         for arg in args:
 463             if arg is None:
 464                 break
 465             if not isinstance(reference, dict):
 466                 raise RoException("URL contains unexpected extra items '{}'".format(arg),
 467                                   HTTPStatus.METHOD_NOT_ALLOWED)
 468
 469             if arg in reference:
 470                 reference = reference[arg]
 471             elif "<ID>" in reference:
 472                 reference = reference["<ID>"]
 473             elif "*" in reference:
 474                 # reference = reference["*"]
 475                 break
 476             else:
 477                 raise RoException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 478         if "TODO" in reference and method in reference["TODO"]:
 479             raise RoException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 480         elif "METHODS" not in reference or method not in reference["METHODS"]:
 481             raise RoException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 482         return reference["ROLE_PERMISSION"] + method.lower()
 483
 484     @staticmethod
 485     def _set_location_header(main_topic, version, topic, id):
 486         """
 487         Insert response header Location with the URL of created item base on URL params
 488         :param main_topic:
 489         :param version:
 490         :param topic:
 491         :param id:
 492         :return: None
 493         """
 494         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 495         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(main_topic, version, topic, id)
 496         return
 497
 498     @cherrypy.expose
 499     def default(self, main_topic=None, version=None, topic=None, _id=None, _id2=None, *args, **kwargs):
 500         token_info = None
 501         outdata = None
 502         _format = None
 503         method = "DONE"
 504         rollback = []
 505         engine_session = None
 506         try:
 507             if not main_topic or not version or not topic:
 508                 raise RoException("URL must contain at least 'main_topic/version/topic'",
 509                                   HTTPStatus.METHOD_NOT_ALLOWED)
 510             if main_topic not in ("admin", "ns",):
 511                 raise RoException("URL main_topic '{}' not supported".format(main_topic),
 512                                   HTTPStatus.METHOD_NOT_ALLOWED)
 513             if version != 'v1':
 514                 raise RoException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 515
 516             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 517                 method = kwargs.pop("METHOD")
 518             else:
 519                 method = cherrypy.request.method
 520
 521             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, _id2, *args,
 522                                                            **kwargs)
 523             # skip token validation if requesting a token
 524             indata = self._format_in(kwargs)
 525             if main_topic != "admin" or topic != "tokens":
 526                 token_info = self.authenticator.authorize(role_permission, _id)
 527             outdata, created_id, done = self.map_operation[role_permission](
 528                 engine_session, indata, version, _id, _id2, *args, *kwargs)
 529             if created_id:
 530                 self._set_location_header(main_topic, version, topic, _id)
 531             cherrypy.response.status = HTTPStatus.ACCEPTED.value if not done else HTTPStatus.OK.value if \
 532                 outdata is not None else HTTPStatus.NO_CONTENT.value
 533             return self._format_out(outdata, token_info, _format)
 534         except Exception as e:
 535             if isinstance(e, (RoException, NsException, DbException, FsException, MsgException, AuthException,
 536                               ValidationError)):
 537                 http_code_value = cherrypy.response.status = e.http_code.value
 538                 http_code_name = e.http_code.name
 539                 cherrypy.log("Exception {}".format(e))
 540             else:
 541                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 542                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 543                 http_code_name = HTTPStatus.BAD_REQUEST.name
 544             if hasattr(outdata, "close"):  # is an open file
 545                 outdata.close()
 546             error_text = str(e)
 547             rollback.reverse()
 548             for rollback_item in rollback:
 549                 try:
 550                     if rollback_item.get("operation") == "set":
 551                         self.ns.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
 552                                            rollback_item["content"], fail_on_empty=False)
 553                     else:
 554                         self.ns.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
 555                                            fail_on_empty=False)
 556                 except Exception as e2:
 557                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
 558                     cherrypy.log(rollback_error_text)
 559                     error_text += ". " + rollback_error_text
 560             # if isinstance(e, MsgException):
 561             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 562             #         engine_topic[:-1], method, error_text)
 563             problem_details = {
 564                 "code": http_code_name,
 565                 "status": http_code_value,
 566                 "detail": error_text,
 567             }
 568             return self._format_out(problem_details, token_info)
 569             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 570         finally:
 571             if token_info:
 572                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 573                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 574                         if outdata.get(logging_id):
 575                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
 576
 577
 578 def _start_service():
 579     """
 580     Callback function called when cherrypy.engine starts
 581     Override configuration with env variables
 582     Set database, storage, message configuration
 583     Init database with admin/admin user password
 584     """
 585     global ro_server, vim_admin_thread
 586     # global vim_threads
 587     cherrypy.log.error("Starting osm_ng_ro")
 588     # update general cherrypy configuration
 589     update_dict = {}
 590
 591     engine_config = cherrypy.tree.apps['/ro'].config
 592     for k, v in environ.items():
 593         if not k.startswith("OSMRO_"):
 594             continue
 595         k1, _, k2 = k[6:].lower().partition("_")
 596         if not k2:
 597             continue
 598         try:
 599             if k1 in ("server", "test", "auth", "log"):
 600                 # update [global] configuration
 601                 update_dict[k1 + '.' + k2] = yaml.safe_load(v)
 602             elif k1 == "static":
 603                 # update [/static] configuration
 604                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 605             elif k1 == "tools":
 606                 # update [/] configuration
 607                 engine_config["/"]["tools." + k2.replace('_', '.')] = yaml.safe_load(v)
 608             elif k1 in ("message", "database", "storage", "authentication"):
 609                 engine_config[k1][k2] = yaml.safe_load(v)
 610
 611         except Exception as e:
 612             raise RoException("Cannot load env '{}': {}".format(k, e))
 613
 614     if update_dict:
 615         cherrypy.config.update(update_dict)
 616         engine_config["global"].update(update_dict)
 617
 618     # logging cherrypy
 619     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 620     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 621     logger_server = logging.getLogger("cherrypy.error")
 622     logger_access = logging.getLogger("cherrypy.access")
 623     logger_cherry = logging.getLogger("cherrypy")
 624     logger_nbi = logging.getLogger("ro")
 625
 626     if "log.file" in engine_config["global"]:
 627         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
 628                                                             maxBytes=100e6, backupCount=9, delay=0)
 629         file_handler.setFormatter(log_formatter_simple)
 630         logger_cherry.addHandler(file_handler)
 631         logger_nbi.addHandler(file_handler)
 632     # log always to standard output
 633     for format_, logger in {"ro.server %(filename)s:%(lineno)s": logger_server,
 634                             "ro.access %(filename)s:%(lineno)s": logger_access,
 635                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
 636                             }.items():
 637         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 638         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
 639         str_handler = logging.StreamHandler()
 640         str_handler.setFormatter(log_formatter_cherry)
 641         logger.addHandler(str_handler)
 642
 643     if engine_config["global"].get("log.level"):
 644         logger_cherry.setLevel(engine_config["global"]["log.level"])
 645         logger_nbi.setLevel(engine_config["global"]["log.level"])
 646     # logging other modules
 647     for k1, logname in {"message": "ro.msg", "database": "ro.db", "storage": "ro.fs"}.items():
 648         engine_config[k1]["logger_name"] = logname
 649         logger_module = logging.getLogger(logname)
 650         if "logfile" in engine_config[k1]:
 651             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
 652                                                                 maxBytes=100e6, backupCount=9, delay=0)
 653             file_handler.setFormatter(log_formatter_simple)
 654             logger_module.addHandler(file_handler)
 655         if "loglevel" in engine_config[k1]:
 656             logger_module.setLevel(engine_config[k1]["loglevel"])
 657     # TODO add more entries, e.g.: storage
 658
 659     engine_config["assignment"] = {}
 660     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 661     cherrypy.tree.apps['/ro'].root.ns.start(engine_config)
 662     cherrypy.tree.apps['/ro'].root.authenticator.start(engine_config)
 663     cherrypy.tree.apps['/ro'].root.ns.init_db(target_version=database_version)
 664
 665     # # start subscriptions thread:
 666     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 667     vim_admin_thread.start()
 668     # # Do not capture except SubscriptionException
 669
 670     # backend = engine_config["authentication"]["backend"]
 671     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 672     #                    .format(ro_version, ro_version_date, backend))
 673
 674
 675 def _stop_service():
 676     """
 677     Callback function called when cherrypy.engine stops
 678     TODO: Ending database connections.
 679     """
 680     global vim_admin_thread
 681     # terminate vim_admin_thread
 682     if vim_admin_thread:
 683         vim_admin_thread.terminate()
 684     vim_admin_thread = None
 685     cherrypy.tree.apps['/ro'].root.ns.stop()
 686     cherrypy.log.error("Stopping osm_ng_ro")
 687
 688
 689 def ro_main(config_file):
 690     global ro_server
 691     ro_server = Server()
 692     cherrypy.engine.subscribe('start', _start_service)
 693     cherrypy.engine.subscribe('stop', _stop_service)
 694     cherrypy.quickstart(ro_server, '/ro', config_file)
 695
 696
 697 def usage():
 698     print("""Usage: {} [options]
 699         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 700         -h|--help: shows this help
 701         """.format(sys.argv[0]))
 702     # --log-socket-host HOST: send logs to this host")
 703     # --log-socket-port PORT: send logs using this port (default: 9022)")
 704
 705
 706 if __name__ == '__main__':
 707     try:
 708         # load parameters and configuration
 709         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 710         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 711         config_file = None
 712         for o, a in opts:
 713             if o in ("-h", "--help"):
 714                 usage()
 715                 sys.exit()
 716             elif o in ("-c", "--config"):
 717                 config_file = a
 718             else:
 719                 assert False, "Unhandled option"
 720         if config_file:
 721             if not path.isfile(config_file):
 722                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
 723                 exit(1)
 724         else:
 725             for config_file in (path.dirname(__file__) + "/ro.cfg", "./ro.cfg", "/etc/osm/ro.cfg"):
 726                 if path.isfile(config_file):
 727                     break
 728             else:
 729                 print("No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
 730                 exit(1)
 731         ro_main(config_file)
 732     except KeyboardInterrupt:
 733         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 734     except getopt.GetoptError as e:
 735         print(str(e), file=sys.stderr)
 736         # usage()
 737         exit(1)