NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 from codecs import getreader
  22 import getopt
  23 from http import HTTPStatus
  24 import json
  25 import logging
  26 import logging.handlers
  27 from os import environ, path
  28 import sys
  29 import time
  30
  31 import cherrypy
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  36 import osm_ng_ro.html_out as html
  37 from osm_ng_ro.monitor import start_monitoring, stop_monitoring
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  44
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "deploy": {
  87                 "METHODS": ("GET",),
  88                 "ROLE_PERMISSION": "deploy:",
  89                 "<ID>": {
  90                     "METHODS": ("GET", "POST", "DELETE"),
  91                     "ROLE_PERMISSION": "deploy:id:",
  92                     "<ID>": {
  93                         "METHODS": ("GET",),
  94                         "ROLE_PERMISSION": "deploy:id:id:",
  95                         "cancel": {
  96                             "METHODS": ("POST",),
  97                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  98                         },
  99                     },
 100                 },
 101             },
 102         }
 103     },
 104 }
 105
 106
 107 class RoException(Exception):
 108     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 109         Exception.__init__(self, message)
 110         self.http_code = http_code
 111
 112
 113 class AuthException(RoException):
 114     pass
 115
 116
 117 class Authenticator:
 118     def __init__(self, valid_url_methods, valid_query_string):
 119         self.valid_url_methods = valid_url_methods
 120         self.valid_query_string = valid_query_string
 121
 122     def authorize(self, *args, **kwargs):
 123         return {"token": "ok", "id": "ok"}
 124
 125     def new_token(self, token_info, indata, remote):
 126         return {"token": "ok", "id": "ok", "remote": remote}
 127
 128     def del_token(self, token_id):
 129         pass
 130
 131     def start(self, engine_config):
 132         pass
 133
 134
 135 class Server(object):
 136     instance = 0
 137     # to decode bytes to str
 138     reader = getreader("utf-8")
 139
 140     def __init__(self):
 141         self.instance += 1
 142         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 143         self.ns = Ns()
 144         self.map_operation = {
 145             "token:post": self.new_token,
 146             "token:id:delete": self.del_token,
 147             "deploy:get": self.ns.get_deploy,
 148             "deploy:id:get": self.ns.get_actions,
 149             "deploy:id:post": self.ns.deploy,
 150             "deploy:id:delete": self.ns.delete,
 151             "deploy:id:id:get": self.ns.status,
 152             "deploy:id:id:cancel:post": self.ns.cancel,
 153         }
 154
 155     def _format_in(self, kwargs):
 156         try:
 157             indata = None
 158
 159             if cherrypy.request.body.length:
 160                 error_text = "Invalid input format "
 161
 162                 if "Content-Type" in cherrypy.request.headers:
 163                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 164                         error_text = "Invalid json format "
 165                         indata = json.load(self.reader(cherrypy.request.body))
 166                         cherrypy.request.headers.pop("Content-File-MD5", None)
 167                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 168                         error_text = "Invalid yaml format "
 169                         indata = yaml.safe_load(cherrypy.request.body)
 170                         cherrypy.request.headers.pop("Content-File-MD5", None)
 171                     elif (
 172                         "application/binary" in cherrypy.request.headers["Content-Type"]
 173                         or "application/gzip"
 174                         in cherrypy.request.headers["Content-Type"]
 175                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 176                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 177                     ):
 178                         indata = cherrypy.request.body  # .read()
 179                     elif (
 180                         "multipart/form-data"
 181                         in cherrypy.request.headers["Content-Type"]
 182                     ):
 183                         if "descriptor_file" in kwargs:
 184                             filecontent = kwargs.pop("descriptor_file")
 185
 186                             if not filecontent.file:
 187                                 raise RoException(
 188                                     "empty file or content", HTTPStatus.BAD_REQUEST
 189                                 )
 190
 191                             indata = filecontent.file  # .read()
 192
 193                             if filecontent.content_type.value:
 194                                 cherrypy.request.headers[
 195                                     "Content-Type"
 196                                 ] = filecontent.content_type.value
 197                     else:
 198                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 199                         #                          "Only 'Content-Type' of type 'application/json' or
 200                         # 'application/yaml' for input format are available")
 201                         error_text = "Invalid yaml format "
 202                         indata = yaml.safe_load(cherrypy.request.body)
 203                         cherrypy.request.headers.pop("Content-File-MD5", None)
 204                 else:
 205                     error_text = "Invalid yaml format "
 206                     indata = yaml.safe_load(cherrypy.request.body)
 207                     cherrypy.request.headers.pop("Content-File-MD5", None)
 208
 209             if not indata:
 210                 indata = {}
 211
 212             format_yaml = False
 213             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 214                 format_yaml = True
 215
 216             for k, v in kwargs.items():
 217                 if isinstance(v, str):
 218                     if v == "":
 219                         kwargs[k] = None
 220                     elif format_yaml:
 221                         try:
 222                             kwargs[k] = yaml.safe_load(v)
 223                         except Exception as yaml_error:
 224                             logging.exception(
 225                                 f"{yaml_error} occured while parsing the yaml"
 226                             )
 227                     elif (
 228                         k.endswith(".gt")
 229                         or k.endswith(".lt")
 230                         or k.endswith(".gte")
 231                         or k.endswith(".lte")
 232                     ):
 233                         try:
 234                             kwargs[k] = int(v)
 235                         except Exception:
 236                             try:
 237                                 kwargs[k] = float(v)
 238                             except Exception as keyword_error:
 239                                 logging.exception(
 240                                     f"{keyword_error} occured while getting the keyword arguments"
 241                                 )
 242                     elif v.find(",") > 0:
 243                         kwargs[k] = v.split(",")
 244                 elif isinstance(v, (list, tuple)):
 245                     for index in range(0, len(v)):
 246                         if v[index] == "":
 247                             v[index] = None
 248                         elif format_yaml:
 249                             try:
 250                                 v[index] = yaml.safe_load(v[index])
 251                             except Exception as error:
 252                                 logging.exception(
 253                                     f"{error} occured while parsing the yaml"
 254                                 )
 255
 256             return indata
 257         except (ValueError, yaml.YAMLError) as exc:
 258             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 259         except KeyError as exc:
 260             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 261         except Exception as exc:
 262             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 263
 264     @staticmethod
 265     def _format_out(data, token_info=None, _format=None):
 266         """
 267         return string of dictionary data according to requested json, yaml, xml. By default json
 268         :param data: response to be sent. Can be a dict, text or file
 269         :param token_info: Contains among other username and project
 270         :param _format: The format to be set as Content-Type if data is a file
 271         :return: None
 272         """
 273         accept = cherrypy.request.headers.get("Accept")
 274
 275         if data is None:
 276             if accept and "text/html" in accept:
 277                 return html.format(
 278                     data, cherrypy.request, cherrypy.response, token_info
 279                 )
 280
 281             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 282             return
 283         elif hasattr(data, "read"):  # file object
 284             if _format:
 285                 cherrypy.response.headers["Content-Type"] = _format
 286             elif "b" in data.mode:  # binariy asssumig zip
 287                 cherrypy.response.headers["Content-Type"] = "application/zip"
 288             else:
 289                 cherrypy.response.headers["Content-Type"] = "text/plain"
 290
 291             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 292             return data
 293
 294         if accept:
 295             if "application/json" in accept:
 296                 cherrypy.response.headers[
 297                     "Content-Type"
 298                 ] = "application/json; charset=utf-8"
 299                 a = json.dumps(data, indent=4) + "\n"
 300
 301                 return a.encode("utf8")
 302             elif "text/html" in accept:
 303                 return html.format(
 304                     data, cherrypy.request, cherrypy.response, token_info
 305                 )
 306             elif (
 307                 "application/yaml" in accept
 308                 or "*/*" in accept
 309                 or "text/plain" in accept
 310             ):
 311                 pass
 312             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 313             elif cherrypy.response.status >= 400:
 314                 raise cherrypy.HTTPError(
 315                     HTTPStatus.NOT_ACCEPTABLE.value,
 316                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 317                     "for output format are available",
 318                 )
 319
 320         cherrypy.response.headers["Content-Type"] = "application/yaml"
 321
 322         return yaml.safe_dump(
 323             data,
 324             explicit_start=True,
 325             indent=4,
 326             default_flow_style=False,
 327             tags=False,
 328             encoding="utf-8",
 329             allow_unicode=True,
 330         )  # , canonical=True, default_style='"'
 331
 332     @cherrypy.expose
 333     def index(self, *args, **kwargs):
 334         token_info = None
 335
 336         try:
 337             if cherrypy.request.method == "GET":
 338                 token_info = self.authenticator.authorize()
 339                 outdata = token_info  # Home page
 340             else:
 341                 raise cherrypy.HTTPError(
 342                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 343                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 344                 )
 345
 346             return self._format_out(outdata, token_info)
 347         except (NsException, AuthException) as e:
 348             # cherrypy.log("index Exception {}".format(e))
 349             cherrypy.response.status = e.http_code.value
 350
 351             return self._format_out("Welcome to OSM!", token_info)
 352
 353     @cherrypy.expose
 354     def version(self, *args, **kwargs):
 355         # TODO consider to remove and provide version using the static version file
 356         try:
 357             if cherrypy.request.method != "GET":
 358                 raise RoException(
 359                     "Only method GET is allowed",
 360                     HTTPStatus.METHOD_NOT_ALLOWED,
 361                 )
 362             elif args or kwargs:
 363                 raise RoException(
 364                     "Invalid URL or query string for version",
 365                     HTTPStatus.METHOD_NOT_ALLOWED,
 366                 )
 367
 368             # TODO include version of other modules, pick up from some kafka admin message
 369             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 370
 371             return self._format_out(osm_ng_ro_version)
 372         except RoException as e:
 373             cherrypy.response.status = e.http_code.value
 374             problem_details = {
 375                 "code": e.http_code.name,
 376                 "status": e.http_code.value,
 377                 "detail": str(e),
 378             }
 379
 380             return self._format_out(problem_details, None)
 381
 382     def new_token(self, engine_session, indata, *args, **kwargs):
 383         token_info = None
 384
 385         try:
 386             token_info = self.authenticator.authorize()
 387         except Exception:
 388             token_info = None
 389
 390         if kwargs:
 391             indata.update(kwargs)
 392
 393         # This is needed to log the user when authentication fails
 394         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 395         token_info = self.authenticator.new_token(
 396             token_info, indata, cherrypy.request.remote
 397         )
 398         cherrypy.session["Authorization"] = token_info["id"]
 399         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 400         # for logging
 401
 402         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 403         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 404
 405         return token_info, token_info["id"], True
 406
 407     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 408         token_id = _id
 409
 410         if not token_id and "id" in kwargs:
 411             token_id = kwargs["id"]
 412         elif not token_id:
 413             token_info = self.authenticator.authorize()
 414             # for logging
 415             token_id = token_info["id"]
 416
 417         self.authenticator.del_token(token_id)
 418         token_info = None
 419         cherrypy.session["Authorization"] = "logout"
 420         # cherrypy.response.cookie["Authorization"] = token_id
 421         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 422
 423         return None, None, True
 424
 425     @cherrypy.expose
 426     def test(self, *args, **kwargs):
 427         if not cherrypy.config.get("server.enable_test") or (
 428             isinstance(cherrypy.config["server.enable_test"], str)
 429             and cherrypy.config["server.enable_test"].lower() == "false"
 430         ):
 431             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 432
 433             return "test URL is disabled"
 434
 435         thread_info = None
 436
 437         if args and args[0] == "help":
 438             return (
 439                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 440                 "sleep/<time>\nmessage/topic\n</pre></html>"
 441             )
 442         elif args and args[0] == "init":
 443             try:
 444                 # self.ns.load_dbase(cherrypy.request.app.config)
 445                 self.ns.create_admin()
 446
 447                 return "Done. User 'admin', password 'admin' created"
 448             except Exception:
 449                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 450
 451                 return self._format_out("Database already initialized")
 452         elif args and args[0] == "file":
 453             return cherrypy.lib.static.serve_file(
 454                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 455                 "text/plain",
 456                 "attachment",
 457             )
 458         elif args and args[0] == "file2":
 459             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 460             f = open(f_path, "r")
 461             cherrypy.response.headers["Content-type"] = "text/plain"
 462             return f
 463
 464         elif len(args) == 2 and args[0] == "db-clear":
 465             deleted_info = self.ns.db.del_list(args[1], kwargs)
 466             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 467         elif len(args) and args[0] == "fs-clear":
 468             if len(args) >= 2:
 469                 folders = (args[1],)
 470             else:
 471                 folders = self.ns.fs.dir_ls(".")
 472
 473             for folder in folders:
 474                 self.ns.fs.file_delete(folder)
 475
 476             return ",".join(folders) + " folders deleted\n"
 477         elif args and args[0] == "login":
 478             if not cherrypy.request.headers.get("Authorization"):
 479                 cherrypy.response.headers[
 480                     "WWW-Authenticate"
 481                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 482                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 483         elif args and args[0] == "login2":
 484             if not cherrypy.request.headers.get("Authorization"):
 485                 cherrypy.response.headers[
 486                     "WWW-Authenticate"
 487                 ] = 'Bearer realm="Access to OSM site"'
 488                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 489         elif args and args[0] == "sleep":
 490             sleep_time = 5
 491
 492             try:
 493                 sleep_time = int(args[1])
 494             except Exception:
 495                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 496                 return self._format_out("Database already initialized")
 497
 498             thread_info = cherrypy.thread_data
 499             print(thread_info)
 500             time.sleep(sleep_time)
 501             # thread_info
 502         elif len(args) >= 2 and args[0] == "message":
 503             main_topic = args[1]
 504             return_text = "<html><pre>{} ->\n".format(main_topic)
 505
 506             try:
 507                 if cherrypy.request.method == "POST":
 508                     to_send = yaml.safe_load(cherrypy.request.body)
 509                     for k, v in to_send.items():
 510                         self.ns.msg.write(main_topic, k, v)
 511                         return_text += "  {}: {}\n".format(k, v)
 512                 elif cherrypy.request.method == "GET":
 513                     for k, v in kwargs.items():
 514                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 515                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 516             except Exception as e:
 517                 return_text += "Error: " + str(e)
 518
 519             return_text += "</pre></html>\n"
 520
 521             return return_text
 522
 523         return_text = (
 524             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 525             + "  kwargs: {}\n".format(kwargs)
 526             + "  headers: {}\n".format(cherrypy.request.headers)
 527             + "  path_info: {}\n".format(cherrypy.request.path_info)
 528             + "  query_string: {}\n".format(cherrypy.request.query_string)
 529             + "  session: {}\n".format(cherrypy.session)
 530             + "  cookie: {}\n".format(cherrypy.request.cookie)
 531             + "  method: {}\n".format(cherrypy.request.method)
 532             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 533             + "  body:\n"
 534         )
 535         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 536
 537         if cherrypy.request.body.length:
 538             return_text += "    content: {}\n".format(
 539                 str(
 540                     cherrypy.request.body.read(
 541                         int(cherrypy.request.headers.get("Content-Length", 0))
 542                     )
 543                 )
 544             )
 545
 546         if thread_info:
 547             return_text += "thread: {}\n".format(thread_info)
 548
 549         return_text += "</pre></html>"
 550
 551         return return_text
 552
 553     @staticmethod
 554     def _check_valid_url_method(method, *args):
 555         if len(args) < 3:
 556             raise RoException(
 557                 "URL must contain at least 'main_topic/version/topic'",
 558                 HTTPStatus.METHOD_NOT_ALLOWED,
 559             )
 560
 561         reference = valid_url_methods
 562         for arg in args:
 563             if arg is None:
 564                 break
 565
 566             if not isinstance(reference, dict):
 567                 raise RoException(
 568                     "URL contains unexpected extra items '{}'".format(arg),
 569                     HTTPStatus.METHOD_NOT_ALLOWED,
 570                 )
 571
 572             if arg in reference:
 573                 reference = reference[arg]
 574             elif "<ID>" in reference:
 575                 reference = reference["<ID>"]
 576             elif "*" in reference:
 577                 # reference = reference["*"]
 578                 break
 579             else:
 580                 raise RoException(
 581                     "Unexpected URL item {}".format(arg),
 582                     HTTPStatus.METHOD_NOT_ALLOWED,
 583                 )
 584
 585         if "TODO" in reference and method in reference["TODO"]:
 586             raise RoException(
 587                 "Method {} not supported yet for this URL".format(method),
 588                 HTTPStatus.NOT_IMPLEMENTED,
 589             )
 590         elif "METHODS" not in reference or method not in reference["METHODS"]:
 591             raise RoException(
 592                 "Method {} not supported for this URL".format(method),
 593                 HTTPStatus.METHOD_NOT_ALLOWED,
 594             )
 595
 596         return reference["ROLE_PERMISSION"] + method.lower()
 597
 598     @staticmethod
 599     def _set_location_header(main_topic, version, topic, id):
 600         """
 601         Insert response header Location with the URL of created item base on URL params
 602         :param main_topic:
 603         :param version:
 604         :param topic:
 605         :param id:
 606         :return: None
 607         """
 608         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 609         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 610             main_topic, version, topic, id
 611         )
 612
 613         return
 614
 615     @cherrypy.expose
 616     def default(
 617         self,
 618         main_topic=None,
 619         version=None,
 620         topic=None,
 621         _id=None,
 622         _id2=None,
 623         *args,
 624         **kwargs,
 625     ):
 626         token_info = None
 627         outdata = None
 628         _format = None
 629         method = "DONE"
 630         rollback = []
 631         engine_session = None
 632
 633         try:
 634             if not main_topic or not version or not topic:
 635                 raise RoException(
 636                     "URL must contain at least 'main_topic/version/topic'",
 637                     HTTPStatus.METHOD_NOT_ALLOWED,
 638                 )
 639
 640             if main_topic not in (
 641                 "admin",
 642                 "ns",
 643             ):
 644                 raise RoException(
 645                     "URL main_topic '{}' not supported".format(main_topic),
 646                     HTTPStatus.METHOD_NOT_ALLOWED,
 647                 )
 648
 649             if version != "v1":
 650                 raise RoException(
 651                     "URL version '{}' not supported".format(version),
 652                     HTTPStatus.METHOD_NOT_ALLOWED,
 653                 )
 654
 655             if (
 656                 kwargs
 657                 and "METHOD" in kwargs
 658                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 659             ):
 660                 method = kwargs.pop("METHOD")
 661             else:
 662                 method = cherrypy.request.method
 663
 664             role_permission = self._check_valid_url_method(
 665                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 666             )
 667             # skip token validation if requesting a token
 668             indata = self._format_in(kwargs)
 669
 670             if main_topic != "admin" or topic != "tokens":
 671                 token_info = self.authenticator.authorize(role_permission, _id)
 672
 673             outdata, created_id, done = self.map_operation[role_permission](
 674                 engine_session, indata, version, _id, _id2, *args, *kwargs
 675             )
 676
 677             if created_id:
 678                 self._set_location_header(main_topic, version, topic, _id)
 679
 680             cherrypy.response.status = (
 681                 HTTPStatus.ACCEPTED.value
 682                 if not done
 683                 else HTTPStatus.OK.value
 684                 if outdata is not None
 685                 else HTTPStatus.NO_CONTENT.value
 686             )
 687
 688             return self._format_out(outdata, token_info, _format)
 689         except Exception as e:
 690             if isinstance(
 691                 e,
 692                 (
 693                     RoException,
 694                     NsException,
 695                     DbException,
 696                     FsException,
 697                     MsgException,
 698                     AuthException,
 699                     ValidationError,
 700                 ),
 701             ):
 702                 http_code_value = cherrypy.response.status = e.http_code.value
 703                 http_code_name = e.http_code.name
 704                 cherrypy.log("Exception {}".format(e))
 705             else:
 706                 http_code_value = (
 707                     cherrypy.response.status
 708                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 709                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 710                 http_code_name = HTTPStatus.BAD_REQUEST.name
 711
 712             if hasattr(outdata, "close"):  # is an open file
 713                 outdata.close()
 714
 715             error_text = str(e)
 716             rollback.reverse()
 717
 718             for rollback_item in rollback:
 719                 try:
 720                     if rollback_item.get("operation") == "set":
 721                         self.ns.db.set_one(
 722                             rollback_item["topic"],
 723                             {"_id": rollback_item["_id"]},
 724                             rollback_item["content"],
 725                             fail_on_empty=False,
 726                         )
 727                     else:
 728                         self.ns.db.del_one(
 729                             rollback_item["topic"],
 730                             {"_id": rollback_item["_id"]},
 731                             fail_on_empty=False,
 732                         )
 733                 except Exception as e2:
 734                     rollback_error_text = "Rollback Exception {}: {}".format(
 735                         rollback_item, e2
 736                     )
 737                     cherrypy.log(rollback_error_text)
 738                     error_text += ". " + rollback_error_text
 739
 740             # if isinstance(e, MsgException):
 741             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 742             #         engine_topic[:-1], method, error_text)
 743             problem_details = {
 744                 "code": http_code_name,
 745                 "status": http_code_value,
 746                 "detail": error_text,
 747             }
 748
 749             return self._format_out(problem_details, token_info)
 750             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 751         finally:
 752             if token_info:
 753                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 754                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 755                         if outdata.get(logging_id):
 756                             cherrypy.request.login += ";{}={}".format(
 757                                 logging_id, outdata[logging_id][:36]
 758                             )
 759
 760
 761 def _start_service():
 762     """
 763     Callback function called when cherrypy.engine starts
 764     Override configuration with env variables
 765     Set database, storage, message configuration
 766     Init database with admin/admin user password
 767     """
 768     global ro_server, vim_admin_thread
 769     # global vim_threads
 770     cherrypy.log.error("Starting osm_ng_ro")
 771     # update general cherrypy configuration
 772     update_dict = {}
 773     engine_config = cherrypy.tree.apps["/ro"].config
 774
 775     for k, v in environ.items():
 776         if not k.startswith("OSMRO_"):
 777             continue
 778
 779         k1, _, k2 = k[6:].lower().partition("_")
 780
 781         if not k2:
 782             continue
 783
 784         try:
 785             if k1 in ("server", "test", "auth", "log"):
 786                 # update [global] configuration
 787                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 788             elif k1 == "static":
 789                 # update [/static] configuration
 790                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 791             elif k1 == "tools":
 792                 # update [/] configuration
 793                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 794             elif k1 in ("message", "database", "storage", "authentication", "period"):
 795                 engine_config[k1][k2] = yaml.safe_load(v)
 796
 797         except Exception as e:
 798             raise RoException("Cannot load env '{}': {}".format(k, e))
 799
 800     if update_dict:
 801         cherrypy.config.update(update_dict)
 802         engine_config["global"].update(update_dict)
 803
 804     # logging cherrypy
 805     log_format_simple = (
 806         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 807     )
 808     log_formatter_simple = logging.Formatter(
 809         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 810     )
 811     logger_server = logging.getLogger("cherrypy.error")
 812     logger_access = logging.getLogger("cherrypy.access")
 813     logger_cherry = logging.getLogger("cherrypy")
 814     logger = logging.getLogger("ro")
 815
 816     if "log.file" in engine_config["global"]:
 817         file_handler = logging.handlers.RotatingFileHandler(
 818             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 819         )
 820         file_handler.setFormatter(log_formatter_simple)
 821         logger_cherry.addHandler(file_handler)
 822         logger.addHandler(file_handler)
 823
 824     # log always to standard output
 825     for format_, logger in {
 826         "ro.server %(filename)s:%(lineno)s": logger_server,
 827         "ro.access %(filename)s:%(lineno)s": logger_access,
 828         "%(name)s %(filename)s:%(lineno)s": logger,
 829     }.items():
 830         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 831         log_formatter_cherry = logging.Formatter(
 832             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 833         )
 834         str_handler = logging.StreamHandler()
 835         str_handler.setFormatter(log_formatter_cherry)
 836         logger.addHandler(str_handler)
 837
 838     if engine_config["global"].get("log.level"):
 839         logger_cherry.setLevel(engine_config["global"]["log.level"])
 840         logger.setLevel(engine_config["global"]["log.level"])
 841
 842     # logging other modules
 843     for k1, logname in {
 844         "message": "ro.msg",
 845         "database": "ro.db",
 846         "storage": "ro.fs",
 847     }.items():
 848         engine_config[k1]["logger_name"] = logname
 849         logger_module = logging.getLogger(logname)
 850
 851         if "logfile" in engine_config[k1]:
 852             file_handler = logging.handlers.RotatingFileHandler(
 853                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 854             )
 855             file_handler.setFormatter(log_formatter_simple)
 856             logger_module.addHandler(file_handler)
 857
 858         if "loglevel" in engine_config[k1]:
 859             logger_module.setLevel(engine_config[k1]["loglevel"])
 860     # TODO add more entries, e.g.: storage
 861
 862     engine_config["assignment"] = {}
 863     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 864     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 865     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 866     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 867
 868     # # start subscriptions thread:
 869     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 870     vim_admin_thread.start()
 871     start_monitoring(config=engine_config)
 872
 873     # # Do not capture except SubscriptionException
 874
 875     # backend = engine_config["authentication"]["backend"]
 876     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 877     #                    .format(ro_version, ro_version_date, backend))
 878
 879
 880 def _stop_service():
 881     """
 882     Callback function called when cherrypy.engine stops
 883     TODO: Ending database connections.
 884     """
 885     global vim_admin_thread
 886
 887     # terminate vim_admin_thread
 888     if vim_admin_thread:
 889         vim_admin_thread.terminate()
 890     stop_monitoring()
 891     vim_admin_thread = None
 892     cherrypy.tree.apps["/ro"].root.ns.stop()
 893     cherrypy.log.error("Stopping osm_ng_ro")
 894
 895
 896 def ro_main(config_file):
 897     global ro_server
 898
 899     ro_server = Server()
 900     cherrypy.engine.subscribe("start", _start_service)
 901     cherrypy.engine.subscribe("stop", _stop_service)
 902     cherrypy.quickstart(ro_server, "/ro", config_file)
 903
 904
 905 def usage():
 906     print(
 907         """Usage: {} [options]
 908         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 909         -h|--help: shows this help
 910         """.format(
 911             sys.argv[0]
 912         )
 913     )
 914     # --log-socket-host HOST: send logs to this host")
 915     # --log-socket-port PORT: send logs using this port (default: 9022)")
 916
 917
 918 if __name__ == "__main__":
 919     try:
 920         # load parameters and configuration
 921         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 922         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 923         config_file = None
 924
 925         for o, a in opts:
 926             if o in ("-h", "--help"):
 927                 usage()
 928                 sys.exit()
 929             elif o in ("-c", "--config"):
 930                 config_file = a
 931             else:
 932                 raise ValueError("Unhandled option")
 933
 934         if config_file:
 935             if not path.isfile(config_file):
 936                 print(
 937                     "configuration file '{}' that not exist".format(config_file),
 938                     file=sys.stderr,
 939                 )
 940                 exit(1)
 941         else:
 942             for config_file in (
 943                 path.dirname(__file__) + "/ro.cfg",
 944                 "./ro.cfg",
 945                 "/etc/osm/ro.cfg",
 946             ):
 947                 if path.isfile(config_file):
 948                     break
 949             else:
 950                 print(
 951                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 952                     file=sys.stderr,
 953                 )
 954                 exit(1)
 955
 956         ro_main(config_file)
 957     except KeyboardInterrupt:
 958         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 959     except getopt.GetoptError as e:
 960         print(str(e), file=sys.stderr)
 961         # usage()
 962         exit(1)