NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 import cherrypy
  22 import time
  23 import json
  24 import yaml
  25 import osm_ng_ro.html_out as html
  26 import logging
  27 import logging.handlers
  28 import getopt
  29 import sys
  30
  31 from osm_ng_ro.ns import Ns, NsException
  32 from osm_ng_ro.validation import ValidationError
  33 from osm_ng_ro.vim_admin import VimAdminThread
  34 from osm_common.dbbase import DbException
  35 from osm_common.fsbase import FsException
  36 from osm_common.msgbase import MsgException
  37 from http import HTTPStatus
  38 from codecs import getreader
  39 from os import environ, path
  40 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  41
  42 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  43
  44 __version__ = "0.1."  # file version, not NBI version
  45 version_date = "May 2020"
  46
  47 database_version = "1.2"
  48 auth_database_version = "1.0"
  49 ro_server = None  # instance of Server class
  50 vim_admin_thread = None  # instance of VimAdminThread class
  51
  52 # vim_threads = None  # instance of VimThread class
  53
  54 """
  55 RO North Bound Interface
  56 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  57         /ns/v1/deploy                                           O
  58             /<nsrs_id>                                          O       O               O
  59                 /<action_id>                                    O
  60                     /cancel                                             O
  61
  62 """
  63
  64 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  65 # ^ Contains possible administrative query string words:
  66 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  67 #           (not owned by my session project).
  68 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  69 #     FORCE=True(by default)|False: Force edition/deletion operations
  70 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  71
  72 valid_url_methods = {
  73     # contains allowed URL and methods, and the role_permission name
  74     "admin": {
  75         "v1": {
  76             "tokens": {
  77                 "METHODS": ("POST",),
  78                 "ROLE_PERMISSION": "tokens:",
  79                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  80             },
  81         }
  82     },
  83     "ns": {
  84         "v1": {
  85             "deploy": {
  86                 "METHODS": ("GET",),
  87                 "ROLE_PERMISSION": "deploy:",
  88                 "<ID>": {
  89                     "METHODS": ("GET", "POST", "DELETE"),
  90                     "ROLE_PERMISSION": "deploy:id:",
  91                     "<ID>": {
  92                         "METHODS": ("GET",),
  93                         "ROLE_PERMISSION": "deploy:id:id:",
  94                         "cancel": {
  95                             "METHODS": ("POST",),
  96                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  97                         },
  98                     },
  99                 },
 100             },
 101         }
 102     },
 103 }
 104
 105
 106 class RoException(Exception):
 107     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 108         Exception.__init__(self, message)
 109         self.http_code = http_code
 110
 111
 112 class AuthException(RoException):
 113     pass
 114
 115
 116 class Authenticator:
 117     def __init__(self, valid_url_methods, valid_query_string):
 118         self.valid_url_methods = valid_url_methods
 119         self.valid_query_string = valid_query_string
 120
 121     def authorize(self, *args, **kwargs):
 122         return {"token": "ok", "id": "ok"}
 123
 124     def new_token(self, token_info, indata, remote):
 125         return {"token": "ok", "id": "ok", "remote": remote}
 126
 127     def del_token(self, token_id):
 128         pass
 129
 130     def start(self, engine_config):
 131         pass
 132
 133
 134 class Server(object):
 135     instance = 0
 136     # to decode bytes to str
 137     reader = getreader("utf-8")
 138
 139     def __init__(self):
 140         self.instance += 1
 141         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 142         self.ns = Ns()
 143         self.map_operation = {
 144             "token:post": self.new_token,
 145             "token:id:delete": self.del_token,
 146             "deploy:get": self.ns.get_deploy,
 147             "deploy:id:get": self.ns.get_actions,
 148             "deploy:id:post": self.ns.deploy,
 149             "deploy:id:delete": self.ns.delete,
 150             "deploy:id:id:get": self.ns.status,
 151             "deploy:id:id:cancel:post": self.ns.cancel,
 152         }
 153
 154     def _format_in(self, kwargs):
 155         try:
 156             indata = None
 157
 158             if cherrypy.request.body.length:
 159                 error_text = "Invalid input format "
 160
 161                 if "Content-Type" in cherrypy.request.headers:
 162                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 163                         error_text = "Invalid json format "
 164                         indata = json.load(self.reader(cherrypy.request.body))
 165                         cherrypy.request.headers.pop("Content-File-MD5", None)
 166                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 167                         error_text = "Invalid yaml format "
 168                         indata = yaml.load(
 169                             cherrypy.request.body, Loader=yaml.SafeLoader
 170                         )
 171                         cherrypy.request.headers.pop("Content-File-MD5", None)
 172                     elif (
 173                         "application/binary" in cherrypy.request.headers["Content-Type"]
 174                         or "application/gzip"
 175                         in cherrypy.request.headers["Content-Type"]
 176                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 177                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 178                     ):
 179                         indata = cherrypy.request.body  # .read()
 180                     elif (
 181                         "multipart/form-data"
 182                         in cherrypy.request.headers["Content-Type"]
 183                     ):
 184                         if "descriptor_file" in kwargs:
 185                             filecontent = kwargs.pop("descriptor_file")
 186
 187                             if not filecontent.file:
 188                                 raise RoException(
 189                                     "empty file or content", HTTPStatus.BAD_REQUEST
 190                                 )
 191
 192                             indata = filecontent.file  # .read()
 193
 194                             if filecontent.content_type.value:
 195                                 cherrypy.request.headers[
 196                                     "Content-Type"
 197                                 ] = filecontent.content_type.value
 198                     else:
 199                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 200                         #                          "Only 'Content-Type' of type 'application/json' or
 201                         # 'application/yaml' for input format are available")
 202                         error_text = "Invalid yaml format "
 203                         indata = yaml.load(
 204                             cherrypy.request.body, Loader=yaml.SafeLoader
 205                         )
 206                         cherrypy.request.headers.pop("Content-File-MD5", None)
 207                 else:
 208                     error_text = "Invalid yaml format "
 209                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 210                     cherrypy.request.headers.pop("Content-File-MD5", None)
 211
 212             if not indata:
 213                 indata = {}
 214
 215             format_yaml = False
 216             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 217                 format_yaml = True
 218
 219             for k, v in kwargs.items():
 220                 if isinstance(v, str):
 221                     if v == "":
 222                         kwargs[k] = None
 223                     elif format_yaml:
 224                         try:
 225                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 226                         except Exception:
 227                             pass
 228                     elif (
 229                         k.endswith(".gt")
 230                         or k.endswith(".lt")
 231                         or k.endswith(".gte")
 232                         or k.endswith(".lte")
 233                     ):
 234                         try:
 235                             kwargs[k] = int(v)
 236                         except Exception:
 237                             try:
 238                                 kwargs[k] = float(v)
 239                             except Exception:
 240                                 pass
 241                     elif v.find(",") > 0:
 242                         kwargs[k] = v.split(",")
 243                 elif isinstance(v, (list, tuple)):
 244                     for index in range(0, len(v)):
 245                         if v[index] == "":
 246                             v[index] = None
 247                         elif format_yaml:
 248                             try:
 249                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 250                             except Exception:
 251                                 pass
 252
 253             return indata
 254         except (ValueError, yaml.YAMLError) as exc:
 255             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 256         except KeyError as exc:
 257             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 258         except Exception as exc:
 259             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 260
 261     @staticmethod
 262     def _format_out(data, token_info=None, _format=None):
 263         """
 264         return string of dictionary data according to requested json, yaml, xml. By default json
 265         :param data: response to be sent. Can be a dict, text or file
 266         :param token_info: Contains among other username and project
 267         :param _format: The format to be set as Content-Type if data is a file
 268         :return: None
 269         """
 270         accept = cherrypy.request.headers.get("Accept")
 271
 272         if data is None:
 273             if accept and "text/html" in accept:
 274                 return html.format(
 275                     data, cherrypy.request, cherrypy.response, token_info
 276                 )
 277
 278             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 279             return
 280         elif hasattr(data, "read"):  # file object
 281             if _format:
 282                 cherrypy.response.headers["Content-Type"] = _format
 283             elif "b" in data.mode:  # binariy asssumig zip
 284                 cherrypy.response.headers["Content-Type"] = "application/zip"
 285             else:
 286                 cherrypy.response.headers["Content-Type"] = "text/plain"
 287
 288             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 289             return data
 290
 291         if accept:
 292             if "application/json" in accept:
 293                 cherrypy.response.headers[
 294                     "Content-Type"
 295                 ] = "application/json; charset=utf-8"
 296                 a = json.dumps(data, indent=4) + "\n"
 297
 298                 return a.encode("utf8")
 299             elif "text/html" in accept:
 300                 return html.format(
 301                     data, cherrypy.request, cherrypy.response, token_info
 302                 )
 303             elif (
 304                 "application/yaml" in accept
 305                 or "*/*" in accept
 306                 or "text/plain" in accept
 307             ):
 308                 pass
 309             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 310             elif cherrypy.response.status >= 400:
 311                 raise cherrypy.HTTPError(
 312                     HTTPStatus.NOT_ACCEPTABLE.value,
 313                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 314                     "for output format are available",
 315                 )
 316
 317         cherrypy.response.headers["Content-Type"] = "application/yaml"
 318
 319         return yaml.safe_dump(
 320             data,
 321             explicit_start=True,
 322             indent=4,
 323             default_flow_style=False,
 324             tags=False,
 325             encoding="utf-8",
 326             allow_unicode=True,
 327         )  # , canonical=True, default_style='"'
 328
 329     @cherrypy.expose
 330     def index(self, *args, **kwargs):
 331         token_info = None
 332
 333         try:
 334             if cherrypy.request.method == "GET":
 335                 token_info = self.authenticator.authorize()
 336                 outdata = token_info  # Home page
 337             else:
 338                 raise cherrypy.HTTPError(
 339                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 340                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 341                 )
 342
 343             return self._format_out(outdata, token_info)
 344         except (NsException, AuthException) as e:
 345             # cherrypy.log("index Exception {}".format(e))
 346             cherrypy.response.status = e.http_code.value
 347
 348             return self._format_out("Welcome to OSM!", token_info)
 349
 350     @cherrypy.expose
 351     def version(self, *args, **kwargs):
 352         # TODO consider to remove and provide version using the static version file
 353         try:
 354             if cherrypy.request.method != "GET":
 355                 raise RoException(
 356                     "Only method GET is allowed",
 357                     HTTPStatus.METHOD_NOT_ALLOWED,
 358                 )
 359             elif args or kwargs:
 360                 raise RoException(
 361                     "Invalid URL or query string for version",
 362                     HTTPStatus.METHOD_NOT_ALLOWED,
 363                 )
 364
 365             # TODO include version of other modules, pick up from some kafka admin message
 366             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 367
 368             return self._format_out(osm_ng_ro_version)
 369         except RoException as e:
 370             cherrypy.response.status = e.http_code.value
 371             problem_details = {
 372                 "code": e.http_code.name,
 373                 "status": e.http_code.value,
 374                 "detail": str(e),
 375             }
 376
 377             return self._format_out(problem_details, None)
 378
 379     def new_token(self, engine_session, indata, *args, **kwargs):
 380         token_info = None
 381
 382         try:
 383             token_info = self.authenticator.authorize()
 384         except Exception:
 385             token_info = None
 386
 387         if kwargs:
 388             indata.update(kwargs)
 389
 390         # This is needed to log the user when authentication fails
 391         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 392         token_info = self.authenticator.new_token(
 393             token_info, indata, cherrypy.request.remote
 394         )
 395         cherrypy.session["Authorization"] = token_info["id"]
 396         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 397         # for logging
 398
 399         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 400         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 401
 402         return token_info, token_info["id"], True
 403
 404     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 405         token_id = _id
 406
 407         if not token_id and "id" in kwargs:
 408             token_id = kwargs["id"]
 409         elif not token_id:
 410             token_info = self.authenticator.authorize()
 411             # for logging
 412             token_id = token_info["id"]
 413
 414         self.authenticator.del_token(token_id)
 415         token_info = None
 416         cherrypy.session["Authorization"] = "logout"
 417         # cherrypy.response.cookie["Authorization"] = token_id
 418         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 419
 420         return None, None, True
 421
 422     @cherrypy.expose
 423     def test(self, *args, **kwargs):
 424         if not cherrypy.config.get("server.enable_test") or (
 425             isinstance(cherrypy.config["server.enable_test"], str)
 426             and cherrypy.config["server.enable_test"].lower() == "false"
 427         ):
 428             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 429
 430             return "test URL is disabled"
 431
 432         thread_info = None
 433
 434         if args and args[0] == "help":
 435             return (
 436                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 437                 "sleep/<time>\nmessage/topic\n</pre></html>"
 438             )
 439         elif args and args[0] == "init":
 440             try:
 441                 # self.ns.load_dbase(cherrypy.request.app.config)
 442                 self.ns.create_admin()
 443
 444                 return "Done. User 'admin', password 'admin' created"
 445             except Exception:
 446                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 447
 448                 return self._format_out("Database already initialized")
 449         elif args and args[0] == "file":
 450             return cherrypy.lib.static.serve_file(
 451                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 452                 "text/plain",
 453                 "attachment",
 454             )
 455         elif args and args[0] == "file2":
 456             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 457             f = open(f_path, "r")
 458             cherrypy.response.headers["Content-type"] = "text/plain"
 459             return f
 460
 461         elif len(args) == 2 and args[0] == "db-clear":
 462             deleted_info = self.ns.db.del_list(args[1], kwargs)
 463             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 464         elif len(args) and args[0] == "fs-clear":
 465             if len(args) >= 2:
 466                 folders = (args[1],)
 467             else:
 468                 folders = self.ns.fs.dir_ls(".")
 469
 470             for folder in folders:
 471                 self.ns.fs.file_delete(folder)
 472
 473             return ",".join(folders) + " folders deleted\n"
 474         elif args and args[0] == "login":
 475             if not cherrypy.request.headers.get("Authorization"):
 476                 cherrypy.response.headers[
 477                     "WWW-Authenticate"
 478                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 479                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 480         elif args and args[0] == "login2":
 481             if not cherrypy.request.headers.get("Authorization"):
 482                 cherrypy.response.headers[
 483                     "WWW-Authenticate"
 484                 ] = 'Bearer realm="Access to OSM site"'
 485                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 486         elif args and args[0] == "sleep":
 487             sleep_time = 5
 488
 489             try:
 490                 sleep_time = int(args[1])
 491             except Exception:
 492                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 493                 return self._format_out("Database already initialized")
 494
 495             thread_info = cherrypy.thread_data
 496             print(thread_info)
 497             time.sleep(sleep_time)
 498             # thread_info
 499         elif len(args) >= 2 and args[0] == "message":
 500             main_topic = args[1]
 501             return_text = "<html><pre>{} ->\n".format(main_topic)
 502
 503             try:
 504                 if cherrypy.request.method == "POST":
 505                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 506                     for k, v in to_send.items():
 507                         self.ns.msg.write(main_topic, k, v)
 508                         return_text += "  {}: {}\n".format(k, v)
 509                 elif cherrypy.request.method == "GET":
 510                     for k, v in kwargs.items():
 511                         self.ns.msg.write(
 512                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 513                         )
 514                         return_text += "  {}: {}\n".format(
 515                             k, yaml.load(v, Loader=yaml.SafeLoader)
 516                         )
 517             except Exception as e:
 518                 return_text += "Error: " + str(e)
 519
 520             return_text += "</pre></html>\n"
 521
 522             return return_text
 523
 524         return_text = (
 525             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 526             + "  kwargs: {}\n".format(kwargs)
 527             + "  headers: {}\n".format(cherrypy.request.headers)
 528             + "  path_info: {}\n".format(cherrypy.request.path_info)
 529             + "  query_string: {}\n".format(cherrypy.request.query_string)
 530             + "  session: {}\n".format(cherrypy.session)
 531             + "  cookie: {}\n".format(cherrypy.request.cookie)
 532             + "  method: {}\n".format(cherrypy.request.method)
 533             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 534             + "  body:\n"
 535         )
 536         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 537
 538         if cherrypy.request.body.length:
 539             return_text += "    content: {}\n".format(
 540                 str(
 541                     cherrypy.request.body.read(
 542                         int(cherrypy.request.headers.get("Content-Length", 0))
 543                     )
 544                 )
 545             )
 546
 547         if thread_info:
 548             return_text += "thread: {}\n".format(thread_info)
 549
 550         return_text += "</pre></html>"
 551
 552         return return_text
 553
 554     @staticmethod
 555     def _check_valid_url_method(method, *args):
 556         if len(args) < 3:
 557             raise RoException(
 558                 "URL must contain at least 'main_topic/version/topic'",
 559                 HTTPStatus.METHOD_NOT_ALLOWED,
 560             )
 561
 562         reference = valid_url_methods
 563         for arg in args:
 564             if arg is None:
 565                 break
 566
 567             if not isinstance(reference, dict):
 568                 raise RoException(
 569                     "URL contains unexpected extra items '{}'".format(arg),
 570                     HTTPStatus.METHOD_NOT_ALLOWED,
 571                 )
 572
 573             if arg in reference:
 574                 reference = reference[arg]
 575             elif "<ID>" in reference:
 576                 reference = reference["<ID>"]
 577             elif "*" in reference:
 578                 # reference = reference["*"]
 579                 break
 580             else:
 581                 raise RoException(
 582                     "Unexpected URL item {}".format(arg),
 583                     HTTPStatus.METHOD_NOT_ALLOWED,
 584                 )
 585
 586         if "TODO" in reference and method in reference["TODO"]:
 587             raise RoException(
 588                 "Method {} not supported yet for this URL".format(method),
 589                 HTTPStatus.NOT_IMPLEMENTED,
 590             )
 591         elif "METHODS" not in reference or method not in reference["METHODS"]:
 592             raise RoException(
 593                 "Method {} not supported for this URL".format(method),
 594                 HTTPStatus.METHOD_NOT_ALLOWED,
 595             )
 596
 597         return reference["ROLE_PERMISSION"] + method.lower()
 598
 599     @staticmethod
 600     def _set_location_header(main_topic, version, topic, id):
 601         """
 602         Insert response header Location with the URL of created item base on URL params
 603         :param main_topic:
 604         :param version:
 605         :param topic:
 606         :param id:
 607         :return: None
 608         """
 609         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 610         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 611             main_topic, version, topic, id
 612         )
 613
 614         return
 615
 616     @cherrypy.expose
 617     def default(
 618         self,
 619         main_topic=None,
 620         version=None,
 621         topic=None,
 622         _id=None,
 623         _id2=None,
 624         *args,
 625         **kwargs,
 626     ):
 627         token_info = None
 628         outdata = None
 629         _format = None
 630         method = "DONE"
 631         rollback = []
 632         engine_session = None
 633
 634         try:
 635             if not main_topic or not version or not topic:
 636                 raise RoException(
 637                     "URL must contain at least 'main_topic/version/topic'",
 638                     HTTPStatus.METHOD_NOT_ALLOWED,
 639                 )
 640
 641             if main_topic not in (
 642                 "admin",
 643                 "ns",
 644             ):
 645                 raise RoException(
 646                     "URL main_topic '{}' not supported".format(main_topic),
 647                     HTTPStatus.METHOD_NOT_ALLOWED,
 648                 )
 649
 650             if version != "v1":
 651                 raise RoException(
 652                     "URL version '{}' not supported".format(version),
 653                     HTTPStatus.METHOD_NOT_ALLOWED,
 654                 )
 655
 656             if (
 657                 kwargs
 658                 and "METHOD" in kwargs
 659                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 660             ):
 661                 method = kwargs.pop("METHOD")
 662             else:
 663                 method = cherrypy.request.method
 664
 665             role_permission = self._check_valid_url_method(
 666                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 667             )
 668             # skip token validation if requesting a token
 669             indata = self._format_in(kwargs)
 670
 671             if main_topic != "admin" or topic != "tokens":
 672                 token_info = self.authenticator.authorize(role_permission, _id)
 673
 674             outdata, created_id, done = self.map_operation[role_permission](
 675                 engine_session, indata, version, _id, _id2, *args, *kwargs
 676             )
 677
 678             if created_id:
 679                 self._set_location_header(main_topic, version, topic, _id)
 680
 681             cherrypy.response.status = (
 682                 HTTPStatus.ACCEPTED.value
 683                 if not done
 684                 else HTTPStatus.OK.value
 685                 if outdata is not None
 686                 else HTTPStatus.NO_CONTENT.value
 687             )
 688
 689             return self._format_out(outdata, token_info, _format)
 690         except Exception as e:
 691             if isinstance(
 692                 e,
 693                 (
 694                     RoException,
 695                     NsException,
 696                     DbException,
 697                     FsException,
 698                     MsgException,
 699                     AuthException,
 700                     ValidationError,
 701                 ),
 702             ):
 703                 http_code_value = cherrypy.response.status = e.http_code.value
 704                 http_code_name = e.http_code.name
 705                 cherrypy.log("Exception {}".format(e))
 706             else:
 707                 http_code_value = (
 708                     cherrypy.response.status
 709                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 710                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 711                 http_code_name = HTTPStatus.BAD_REQUEST.name
 712
 713             if hasattr(outdata, "close"):  # is an open file
 714                 outdata.close()
 715
 716             error_text = str(e)
 717             rollback.reverse()
 718
 719             for rollback_item in rollback:
 720                 try:
 721                     if rollback_item.get("operation") == "set":
 722                         self.ns.db.set_one(
 723                             rollback_item["topic"],
 724                             {"_id": rollback_item["_id"]},
 725                             rollback_item["content"],
 726                             fail_on_empty=False,
 727                         )
 728                     else:
 729                         self.ns.db.del_one(
 730                             rollback_item["topic"],
 731                             {"_id": rollback_item["_id"]},
 732                             fail_on_empty=False,
 733                         )
 734                 except Exception as e2:
 735                     rollback_error_text = "Rollback Exception {}: {}".format(
 736                         rollback_item, e2
 737                     )
 738                     cherrypy.log(rollback_error_text)
 739                     error_text += ". " + rollback_error_text
 740
 741             # if isinstance(e, MsgException):
 742             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 743             #         engine_topic[:-1], method, error_text)
 744             problem_details = {
 745                 "code": http_code_name,
 746                 "status": http_code_value,
 747                 "detail": error_text,
 748             }
 749
 750             return self._format_out(problem_details, token_info)
 751             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 752         finally:
 753             if token_info:
 754                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 755                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 756                         if outdata.get(logging_id):
 757                             cherrypy.request.login += ";{}={}".format(
 758                                 logging_id, outdata[logging_id][:36]
 759                             )
 760
 761
 762 def _start_service():
 763     """
 764     Callback function called when cherrypy.engine starts
 765     Override configuration with env variables
 766     Set database, storage, message configuration
 767     Init database with admin/admin user password
 768     """
 769     global ro_server, vim_admin_thread
 770     # global vim_threads
 771     cherrypy.log.error("Starting osm_ng_ro")
 772     # update general cherrypy configuration
 773     update_dict = {}
 774     engine_config = cherrypy.tree.apps["/ro"].config
 775
 776     for k, v in environ.items():
 777         if not k.startswith("OSMRO_"):
 778             continue
 779
 780         k1, _, k2 = k[6:].lower().partition("_")
 781
 782         if not k2:
 783             continue
 784
 785         try:
 786             if k1 in ("server", "test", "auth", "log"):
 787                 # update [global] configuration
 788                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 789             elif k1 == "static":
 790                 # update [/static] configuration
 791                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 792             elif k1 == "tools":
 793                 # update [/] configuration
 794                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 795             elif k1 in ("message", "database", "storage", "authentication"):
 796                 engine_config[k1][k2] = yaml.safe_load(v)
 797
 798         except Exception as e:
 799             raise RoException("Cannot load env '{}': {}".format(k, e))
 800
 801     if update_dict:
 802         cherrypy.config.update(update_dict)
 803         engine_config["global"].update(update_dict)
 804
 805     # logging cherrypy
 806     log_format_simple = (
 807         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 808     )
 809     log_formatter_simple = logging.Formatter(
 810         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 811     )
 812     logger_server = logging.getLogger("cherrypy.error")
 813     logger_access = logging.getLogger("cherrypy.access")
 814     logger_cherry = logging.getLogger("cherrypy")
 815     logger = logging.getLogger("ro")
 816
 817     if "log.file" in engine_config["global"]:
 818         file_handler = logging.handlers.RotatingFileHandler(
 819             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 820         )
 821         file_handler.setFormatter(log_formatter_simple)
 822         logger_cherry.addHandler(file_handler)
 823         logger.addHandler(file_handler)
 824
 825     # log always to standard output
 826     for format_, logger in {
 827         "ro.server %(filename)s:%(lineno)s": logger_server,
 828         "ro.access %(filename)s:%(lineno)s": logger_access,
 829         "%(name)s %(filename)s:%(lineno)s": logger,
 830     }.items():
 831         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 832         log_formatter_cherry = logging.Formatter(
 833             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 834         )
 835         str_handler = logging.StreamHandler()
 836         str_handler.setFormatter(log_formatter_cherry)
 837         logger.addHandler(str_handler)
 838
 839     if engine_config["global"].get("log.level"):
 840         logger_cherry.setLevel(engine_config["global"]["log.level"])
 841         logger.setLevel(engine_config["global"]["log.level"])
 842
 843     # logging other modules
 844     for k1, logname in {
 845         "message": "ro.msg",
 846         "database": "ro.db",
 847         "storage": "ro.fs",
 848     }.items():
 849         engine_config[k1]["logger_name"] = logname
 850         logger_module = logging.getLogger(logname)
 851
 852         if "logfile" in engine_config[k1]:
 853             file_handler = logging.handlers.RotatingFileHandler(
 854                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 855             )
 856             file_handler.setFormatter(log_formatter_simple)
 857             logger_module.addHandler(file_handler)
 858
 859         if "loglevel" in engine_config[k1]:
 860             logger_module.setLevel(engine_config[k1]["loglevel"])
 861     # TODO add more entries, e.g.: storage
 862
 863     engine_config["assignment"] = {}
 864     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 865     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 866     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 867     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 868
 869     # # start subscriptions thread:
 870     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 871     vim_admin_thread.start()
 872     # # Do not capture except SubscriptionException
 873
 874     # backend = engine_config["authentication"]["backend"]
 875     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 876     #                    .format(ro_version, ro_version_date, backend))
 877
 878
 879 def _stop_service():
 880     """
 881     Callback function called when cherrypy.engine stops
 882     TODO: Ending database connections.
 883     """
 884     global vim_admin_thread
 885
 886     # terminate vim_admin_thread
 887     if vim_admin_thread:
 888         vim_admin_thread.terminate()
 889
 890     vim_admin_thread = None
 891     cherrypy.tree.apps["/ro"].root.ns.stop()
 892     cherrypy.log.error("Stopping osm_ng_ro")
 893
 894
 895 def ro_main(config_file):
 896     global ro_server
 897
 898     ro_server = Server()
 899     cherrypy.engine.subscribe("start", _start_service)
 900     cherrypy.engine.subscribe("stop", _stop_service)
 901     cherrypy.quickstart(ro_server, "/ro", config_file)
 902
 903
 904 def usage():
 905     print(
 906         """Usage: {} [options]
 907         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 908         -h|--help: shows this help
 909         """.format(
 910             sys.argv[0]
 911         )
 912     )
 913     # --log-socket-host HOST: send logs to this host")
 914     # --log-socket-port PORT: send logs using this port (default: 9022)")
 915
 916
 917 if __name__ == "__main__":
 918     try:
 919         # load parameters and configuration
 920         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 921         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 922         config_file = None
 923
 924         for o, a in opts:
 925             if o in ("-h", "--help"):
 926                 usage()
 927                 sys.exit()
 928             elif o in ("-c", "--config"):
 929                 config_file = a
 930             else:
 931                 assert False, "Unhandled option"
 932
 933         if config_file:
 934             if not path.isfile(config_file):
 935                 print(
 936                     "configuration file '{}' that not exist".format(config_file),
 937                     file=sys.stderr,
 938                 )
 939                 exit(1)
 940         else:
 941             for config_file in (
 942                 path.dirname(__file__) + "/ro.cfg",
 943                 "./ro.cfg",
 944                 "/etc/osm/ro.cfg",
 945             ):
 946                 if path.isfile(config_file):
 947                     break
 948             else:
 949                 print(
 950                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 951                     file=sys.stderr,
 952                 )
 953                 exit(1)
 954
 955         ro_main(config_file)
 956     except KeyboardInterrupt:
 957         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 958     except getopt.GetoptError as e:
 959         print(str(e), file=sys.stderr)
 960         # usage()
 961         exit(1)