NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 from codecs import getreader
  22 import getopt
  23 from http import HTTPStatus
  24 import json
  25 import logging
  26 import logging.handlers
  27 from os import environ, path
  28 import sys
  29 import time
  30
  31 import cherrypy
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  36 import osm_ng_ro.html_out as html
  37 from osm_ng_ro.ns import Ns, NsException
  38 from osm_ng_ro.validation import ValidationError
  39 from osm_ng_ro.vim_admin import VimAdminThread
  40 import yaml
  41
  42 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  43
  44 __version__ = "0.1."  # file version, not NBI version
  45 version_date = "May 2020"
  46
  47 database_version = "1.2"
  48 auth_database_version = "1.0"
  49 ro_server = None  # instance of Server class
  50 vim_admin_thread = None  # instance of VimAdminThread class
  51
  52 # vim_threads = None  # instance of VimThread class
  53
  54 """
  55 RO North Bound Interface
  56 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  57         /ns/v1/deploy                                           O
  58             /<nsrs_id>                                          O       O               O
  59                 /<action_id>                                    O
  60                     /cancel                                             O
  61
  62 """
  63
  64 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  65 # ^ Contains possible administrative query string words:
  66 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  67 #           (not owned by my session project).
  68 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  69 #     FORCE=True(by default)|False: Force edition/deletion operations
  70 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  71
  72 valid_url_methods = {
  73     # contains allowed URL and methods, and the role_permission name
  74     "admin": {
  75         "v1": {
  76             "tokens": {
  77                 "METHODS": ("POST",),
  78                 "ROLE_PERMISSION": "tokens:",
  79                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  80             },
  81         }
  82     },
  83     "ns": {
  84         "v1": {
  85             "deploy": {
  86                 "METHODS": ("GET",),
  87                 "ROLE_PERMISSION": "deploy:",
  88                 "<ID>": {
  89                     "METHODS": ("GET", "POST", "DELETE"),
  90                     "ROLE_PERMISSION": "deploy:id:",
  91                     "<ID>": {
  92                         "METHODS": ("GET",),
  93                         "ROLE_PERMISSION": "deploy:id:id:",
  94                         "cancel": {
  95                             "METHODS": ("POST",),
  96                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  97                         },
  98                     },
  99                 },
 100             },
 101         }
 102     },
 103 }
 104
 105
 106 class RoException(Exception):
 107     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 108         Exception.__init__(self, message)
 109         self.http_code = http_code
 110
 111
 112 class AuthException(RoException):
 113     pass
 114
 115
 116 class Authenticator:
 117     def __init__(self, valid_url_methods, valid_query_string):
 118         self.valid_url_methods = valid_url_methods
 119         self.valid_query_string = valid_query_string
 120
 121     def authorize(self, *args, **kwargs):
 122         return {"token": "ok", "id": "ok"}
 123
 124     def new_token(self, token_info, indata, remote):
 125         return {"token": "ok", "id": "ok", "remote": remote}
 126
 127     def del_token(self, token_id):
 128         pass
 129
 130     def start(self, engine_config):
 131         pass
 132
 133
 134 class Server(object):
 135     instance = 0
 136     # to decode bytes to str
 137     reader = getreader("utf-8")
 138
 139     def __init__(self):
 140         self.instance += 1
 141         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 142         self.ns = Ns()
 143         self.map_operation = {
 144             "token:post": self.new_token,
 145             "token:id:delete": self.del_token,
 146             "deploy:get": self.ns.get_deploy,
 147             "deploy:id:get": self.ns.get_actions,
 148             "deploy:id:post": self.ns.deploy,
 149             "deploy:id:delete": self.ns.delete,
 150             "deploy:id:id:get": self.ns.status,
 151             "deploy:id:id:cancel:post": self.ns.cancel,
 152         }
 153
 154     def _format_in(self, kwargs):
 155         try:
 156             indata = None
 157
 158             if cherrypy.request.body.length:
 159                 error_text = "Invalid input format "
 160
 161                 if "Content-Type" in cherrypy.request.headers:
 162                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 163                         error_text = "Invalid json format "
 164                         indata = json.load(self.reader(cherrypy.request.body))
 165                         cherrypy.request.headers.pop("Content-File-MD5", None)
 166                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 167                         error_text = "Invalid yaml format "
 168                         indata = yaml.load(
 169                             cherrypy.request.body, Loader=yaml.SafeLoader
 170                         )
 171                         cherrypy.request.headers.pop("Content-File-MD5", None)
 172                     elif (
 173                         "application/binary" in cherrypy.request.headers["Content-Type"]
 174                         or "application/gzip"
 175                         in cherrypy.request.headers["Content-Type"]
 176                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 177                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 178                     ):
 179                         indata = cherrypy.request.body  # .read()
 180                     elif (
 181                         "multipart/form-data"
 182                         in cherrypy.request.headers["Content-Type"]
 183                     ):
 184                         if "descriptor_file" in kwargs:
 185                             filecontent = kwargs.pop("descriptor_file")
 186
 187                             if not filecontent.file:
 188                                 raise RoException(
 189                                     "empty file or content", HTTPStatus.BAD_REQUEST
 190                                 )
 191
 192                             indata = filecontent.file  # .read()
 193
 194                             if filecontent.content_type.value:
 195                                 cherrypy.request.headers[
 196                                     "Content-Type"
 197                                 ] = filecontent.content_type.value
 198                     else:
 199                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 200                         #                          "Only 'Content-Type' of type 'application/json' or
 201                         # 'application/yaml' for input format are available")
 202                         error_text = "Invalid yaml format "
 203                         indata = yaml.load(
 204                             cherrypy.request.body, Loader=yaml.SafeLoader
 205                         )
 206                         cherrypy.request.headers.pop("Content-File-MD5", None)
 207                 else:
 208                     error_text = "Invalid yaml format "
 209                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 210                     cherrypy.request.headers.pop("Content-File-MD5", None)
 211
 212             if not indata:
 213                 indata = {}
 214
 215             format_yaml = False
 216             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 217                 format_yaml = True
 218
 219             for k, v in kwargs.items():
 220                 if isinstance(v, str):
 221                     if v == "":
 222                         kwargs[k] = None
 223                     elif format_yaml:
 224                         try:
 225                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 226                         except Exception:
 227                             pass
 228                     elif (
 229                         k.endswith(".gt")
 230                         or k.endswith(".lt")
 231                         or k.endswith(".gte")
 232                         or k.endswith(".lte")
 233                     ):
 234                         try:
 235                             kwargs[k] = int(v)
 236                         except Exception:
 237                             try:
 238                                 kwargs[k] = float(v)
 239                             except Exception:
 240                                 pass
 241                     elif v.find(",") > 0:
 242                         kwargs[k] = v.split(",")
 243                 elif isinstance(v, (list, tuple)):
 244                     for index in range(0, len(v)):
 245                         if v[index] == "":
 246                             v[index] = None
 247                         elif format_yaml:
 248                             try:
 249                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 250                             except Exception:
 251                                 pass
 252
 253             return indata
 254         except (ValueError, yaml.YAMLError) as exc:
 255             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 256         except KeyError as exc:
 257             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 258         except Exception as exc:
 259             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 260
 261     @staticmethod
 262     def _format_out(data, token_info=None, _format=None):
 263         """
 264         return string of dictionary data according to requested json, yaml, xml. By default json
 265         :param data: response to be sent. Can be a dict, text or file
 266         :param token_info: Contains among other username and project
 267         :param _format: The format to be set as Content-Type if data is a file
 268         :return: None
 269         """
 270         accept = cherrypy.request.headers.get("Accept")
 271
 272         if data is None:
 273             if accept and "text/html" in accept:
 274                 return html.format(
 275                     data, cherrypy.request, cherrypy.response, token_info
 276                 )
 277
 278             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 279             return
 280         elif hasattr(data, "read"):  # file object
 281             if _format:
 282                 cherrypy.response.headers["Content-Type"] = _format
 283             elif "b" in data.mode:  # binariy asssumig zip
 284                 cherrypy.response.headers["Content-Type"] = "application/zip"
 285             else:
 286                 cherrypy.response.headers["Content-Type"] = "text/plain"
 287
 288             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 289             return data
 290
 291         if accept:
 292             if "application/json" in accept:
 293                 cherrypy.response.headers[
 294                     "Content-Type"
 295                 ] = "application/json; charset=utf-8"
 296                 a = json.dumps(data, indent=4) + "\n"
 297
 298                 return a.encode("utf8")
 299             elif "text/html" in accept:
 300                 return html.format(
 301                     data, cherrypy.request, cherrypy.response, token_info
 302                 )
 303             elif (
 304                 "application/yaml" in accept
 305                 or "*/*" in accept
 306                 or "text/plain" in accept
 307             ):
 308                 pass
 309             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 310             elif cherrypy.response.status >= 400:
 311                 raise cherrypy.HTTPError(
 312                     HTTPStatus.NOT_ACCEPTABLE.value,
 313                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 314                     "for output format are available",
 315                 )
 316
 317         cherrypy.response.headers["Content-Type"] = "application/yaml"
 318
 319         return yaml.safe_dump(
 320             data,
 321             explicit_start=True,
 322             indent=4,
 323             default_flow_style=False,
 324             tags=False,
 325             encoding="utf-8",
 326             allow_unicode=True,
 327         )  # , canonical=True, default_style='"'
 328
 329     @cherrypy.expose
 330     def index(self, *args, **kwargs):
 331         token_info = None
 332
 333         try:
 334             if cherrypy.request.method == "GET":
 335                 token_info = self.authenticator.authorize()
 336                 outdata = token_info  # Home page
 337             else:
 338                 raise cherrypy.HTTPError(
 339                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 340                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 341                 )
 342
 343             return self._format_out(outdata, token_info)
 344         except (NsException, AuthException) as e:
 345             # cherrypy.log("index Exception {}".format(e))
 346             cherrypy.response.status = e.http_code.value
 347
 348             return self._format_out("Welcome to OSM!", token_info)
 349
 350     @cherrypy.expose
 351     def version(self, *args, **kwargs):
 352         # TODO consider to remove and provide version using the static version file
 353         try:
 354             if cherrypy.request.method != "GET":
 355                 raise RoException(
 356                     "Only method GET is allowed",
 357                     HTTPStatus.METHOD_NOT_ALLOWED,
 358                 )
 359             elif args or kwargs:
 360                 raise RoException(
 361                     "Invalid URL or query string for version",
 362                     HTTPStatus.METHOD_NOT_ALLOWED,
 363                 )
 364
 365             # TODO include version of other modules, pick up from some kafka admin message
 366             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 367
 368             return self._format_out(osm_ng_ro_version)
 369         except RoException as e:
 370             cherrypy.response.status = e.http_code.value
 371             problem_details = {
 372                 "code": e.http_code.name,
 373                 "status": e.http_code.value,
 374                 "detail": str(e),
 375             }
 376
 377             return self._format_out(problem_details, None)
 378
 379     def new_token(self, engine_session, indata, *args, **kwargs):
 380         token_info = None
 381
 382         try:
 383             token_info = self.authenticator.authorize()
 384         except Exception:
 385             token_info = None
 386
 387         if kwargs:
 388             indata.update(kwargs)
 389
 390         # This is needed to log the user when authentication fails
 391         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 392         token_info = self.authenticator.new_token(
 393             token_info, indata, cherrypy.request.remote
 394         )
 395         cherrypy.session["Authorization"] = token_info["id"]
 396         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 397         # for logging
 398
 399         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 400         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 401
 402         return token_info, token_info["id"], True
 403
 404     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 405         token_id = _id
 406
 407         if not token_id and "id" in kwargs:
 408             token_id = kwargs["id"]
 409         elif not token_id:
 410             token_info = self.authenticator.authorize()
 411             # for logging
 412             token_id = token_info["id"]
 413
 414         self.authenticator.del_token(token_id)
 415         token_info = None
 416         cherrypy.session["Authorization"] = "logout"
 417         # cherrypy.response.cookie["Authorization"] = token_id
 418         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 419
 420         return None, None, True
 421
 422     @cherrypy.expose
 423     def test(self, *args, **kwargs):
 424         if not cherrypy.config.get("server.enable_test") or (
 425             isinstance(cherrypy.config["server.enable_test"], str)
 426             and cherrypy.config["server.enable_test"].lower() == "false"
 427         ):
 428             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 429
 430             return "test URL is disabled"
 431
 432         thread_info = None
 433
 434         if args and args[0] == "help":
 435             return (
 436                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 437                 "sleep/<time>\nmessage/topic\n</pre></html>"
 438             )
 439         elif args and args[0] == "init":
 440             try:
 441                 # self.ns.load_dbase(cherrypy.request.app.config)
 442                 self.ns.create_admin()
 443
 444                 return "Done. User 'admin', password 'admin' created"
 445             except Exception:
 446                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 447
 448                 return self._format_out("Database already initialized")
 449         elif args and args[0] == "file":
 450             return cherrypy.lib.static.serve_file(
 451                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 452                 "text/plain",
 453                 "attachment",
 454             )
 455         elif args and args[0] == "file2":
 456             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 457             f = open(f_path, "r")
 458             cherrypy.response.headers["Content-type"] = "text/plain"
 459             return f
 460
 461         elif len(args) == 2 and args[0] == "db-clear":
 462             deleted_info = self.ns.db.del_list(args[1], kwargs)
 463             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 464         elif len(args) and args[0] == "fs-clear":
 465             if len(args) >= 2:
 466                 folders = (args[1],)
 467             else:
 468                 folders = self.ns.fs.dir_ls(".")
 469
 470             for folder in folders:
 471                 self.ns.fs.file_delete(folder)
 472
 473             return ",".join(folders) + " folders deleted\n"
 474         elif args and args[0] == "login":
 475             if not cherrypy.request.headers.get("Authorization"):
 476                 cherrypy.response.headers[
 477                     "WWW-Authenticate"
 478                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 479                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 480         elif args and args[0] == "login2":
 481             if not cherrypy.request.headers.get("Authorization"):
 482                 cherrypy.response.headers[
 483                     "WWW-Authenticate"
 484                 ] = 'Bearer realm="Access to OSM site"'
 485                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 486         elif args and args[0] == "sleep":
 487             sleep_time = 5
 488
 489             try:
 490                 sleep_time = int(args[1])
 491             except Exception:
 492                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 493                 return self._format_out("Database already initialized")
 494
 495             thread_info = cherrypy.thread_data
 496             print(thread_info)
 497             time.sleep(sleep_time)
 498             # thread_info
 499         elif len(args) >= 2 and args[0] == "message":
 500             main_topic = args[1]
 501             return_text = "<html><pre>{} ->\n".format(main_topic)
 502
 503             try:
 504                 if cherrypy.request.method == "POST":
 505                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 506                     for k, v in to_send.items():
 507                         self.ns.msg.write(main_topic, k, v)
 508                         return_text += "  {}: {}\n".format(k, v)
 509                 elif cherrypy.request.method == "GET":
 510                     for k, v in kwargs.items():
 511                         self.ns.msg.write(
 512                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 513                         )
 514                         return_text += "  {}: {}\n".format(
 515                             k, yaml.load(v, Loader=yaml.SafeLoader)
 516                         )
 517             except Exception as e:
 518                 return_text += "Error: " + str(e)
 519
 520             return_text += "</pre></html>\n"
 521
 522             return return_text
 523
 524         return_text = (
 525             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 526             + "  kwargs: {}\n".format(kwargs)
 527             + "  headers: {}\n".format(cherrypy.request.headers)
 528             + "  path_info: {}\n".format(cherrypy.request.path_info)
 529             + "  query_string: {}\n".format(cherrypy.request.query_string)
 530             + "  session: {}\n".format(cherrypy.session)
 531             + "  cookie: {}\n".format(cherrypy.request.cookie)
 532             + "  method: {}\n".format(cherrypy.request.method)
 533             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 534             + "  body:\n"
 535         )
 536         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 537
 538         if cherrypy.request.body.length:
 539             return_text += "    content: {}\n".format(
 540                 str(
 541                     cherrypy.request.body.read(
 542                         int(cherrypy.request.headers.get("Content-Length", 0))
 543                     )
 544                 )
 545             )
 546
 547         if thread_info:
 548             return_text += "thread: {}\n".format(thread_info)
 549
 550         return_text += "</pre></html>"
 551
 552         return return_text
 553
 554     @staticmethod
 555     def _check_valid_url_method(method, *args):
 556         if len(args) < 3:
 557             raise RoException(
 558                 "URL must contain at least 'main_topic/version/topic'",
 559                 HTTPStatus.METHOD_NOT_ALLOWED,
 560             )
 561
 562         reference = valid_url_methods
 563         for arg in args:
 564             if arg is None:
 565                 break
 566
 567             if not isinstance(reference, dict):
 568                 raise RoException(
 569                     "URL contains unexpected extra items '{}'".format(arg),
 570                     HTTPStatus.METHOD_NOT_ALLOWED,
 571                 )
 572
 573             if arg in reference:
 574                 reference = reference[arg]
 575             elif "<ID>" in reference:
 576                 reference = reference["<ID>"]
 577             elif "*" in reference:
 578                 # reference = reference["*"]
 579                 break
 580             else:
 581                 raise RoException(
 582                     "Unexpected URL item {}".format(arg),
 583                     HTTPStatus.METHOD_NOT_ALLOWED,
 584                 )
 585
 586         if "TODO" in reference and method in reference["TODO"]:
 587             raise RoException(
 588                 "Method {} not supported yet for this URL".format(method),
 589                 HTTPStatus.NOT_IMPLEMENTED,
 590             )
 591         elif "METHODS" not in reference or method not in reference["METHODS"]:
 592             raise RoException(
 593                 "Method {} not supported for this URL".format(method),
 594                 HTTPStatus.METHOD_NOT_ALLOWED,
 595             )
 596
 597         return reference["ROLE_PERMISSION"] + method.lower()
 598
 599     @staticmethod
 600     def _set_location_header(main_topic, version, topic, id):
 601         """
 602         Insert response header Location with the URL of created item base on URL params
 603         :param main_topic:
 604         :param version:
 605         :param topic:
 606         :param id:
 607         :return: None
 608         """
 609         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 610         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 611             main_topic, version, topic, id
 612         )
 613
 614         return
 615
 616     @cherrypy.expose
 617     def default(
 618         self,
 619         main_topic=None,
 620         version=None,
 621         topic=None,
 622         _id=None,
 623         _id2=None,
 624         *args,
 625         **kwargs,
 626     ):
 627         token_info = None
 628         outdata = None
 629         _format = None
 630         method = "DONE"
 631         rollback = []
 632         engine_session = None
 633
 634         try:
 635             if not main_topic or not version or not topic:
 636                 raise RoException(
 637                     "URL must contain at least 'main_topic/version/topic'",
 638                     HTTPStatus.METHOD_NOT_ALLOWED,
 639                 )
 640
 641             if main_topic not in (
 642                 "admin",
 643                 "ns",
 644             ):
 645                 raise RoException(
 646                     "URL main_topic '{}' not supported".format(main_topic),
 647                     HTTPStatus.METHOD_NOT_ALLOWED,
 648                 )
 649
 650             if version != "v1":
 651                 raise RoException(
 652                     "URL version '{}' not supported".format(version),
 653                     HTTPStatus.METHOD_NOT_ALLOWED,
 654                 )
 655
 656             if (
 657                 kwargs
 658                 and "METHOD" in kwargs
 659                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 660             ):
 661                 method = kwargs.pop("METHOD")
 662             else:
 663                 method = cherrypy.request.method
 664
 665             role_permission = self._check_valid_url_method(
 666                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 667             )
 668             # skip token validation if requesting a token
 669             indata = self._format_in(kwargs)
 670
 671             if main_topic != "admin" or topic != "tokens":
 672                 token_info = self.authenticator.authorize(role_permission, _id)
 673
 674             outdata, created_id, done = self.map_operation[role_permission](
 675                 engine_session, indata, version, _id, _id2, *args, *kwargs
 676             )
 677
 678             if created_id:
 679                 self._set_location_header(main_topic, version, topic, _id)
 680
 681             cherrypy.response.status = (
 682                 HTTPStatus.ACCEPTED.value
 683                 if not done
 684                 else HTTPStatus.OK.value
 685                 if outdata is not None
 686                 else HTTPStatus.NO_CONTENT.value
 687             )
 688
 689             return self._format_out(outdata, token_info, _format)
 690         except Exception as e:
 691             if isinstance(
 692                 e,
 693                 (
 694                     RoException,
 695                     NsException,
 696                     DbException,
 697                     FsException,
 698                     MsgException,
 699                     AuthException,
 700                     ValidationError,
 701                 ),
 702             ):
 703                 http_code_value = cherrypy.response.status = e.http_code.value
 704                 http_code_name = e.http_code.name
 705                 cherrypy.log("Exception {}".format(e))
 706             else:
 707                 http_code_value = (
 708                     cherrypy.response.status
 709                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 710                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 711                 http_code_name = HTTPStatus.BAD_REQUEST.name
 712
 713             if hasattr(outdata, "close"):  # is an open file
 714                 outdata.close()
 715
 716             error_text = str(e)
 717             rollback.reverse()
 718
 719             for rollback_item in rollback:
 720                 try:
 721                     if rollback_item.get("operation") == "set":
 722                         self.ns.db.set_one(
 723                             rollback_item["topic"],
 724                             {"_id": rollback_item["_id"]},
 725                             rollback_item["content"],
 726                             fail_on_empty=False,
 727                         )
 728                     else:
 729                         self.ns.db.del_one(
 730                             rollback_item["topic"],
 731                             {"_id": rollback_item["_id"]},
 732                             fail_on_empty=False,
 733                         )
 734                 except Exception as e2:
 735                     rollback_error_text = "Rollback Exception {}: {}".format(
 736                         rollback_item, e2
 737                     )
 738                     cherrypy.log(rollback_error_text)
 739                     error_text += ". " + rollback_error_text
 740
 741             # if isinstance(e, MsgException):
 742             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 743             #         engine_topic[:-1], method, error_text)
 744             problem_details = {
 745                 "code": http_code_name,
 746                 "status": http_code_value,
 747                 "detail": error_text,
 748             }
 749
 750             return self._format_out(problem_details, token_info)
 751             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 752         finally:
 753             if token_info:
 754                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 755                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 756                         if outdata.get(logging_id):
 757                             cherrypy.request.login += ";{}={}".format(
 758                                 logging_id, outdata[logging_id][:36]
 759                             )
 760
 761
 762 def _start_service():
 763     """
 764     Callback function called when cherrypy.engine starts
 765     Override configuration with env variables
 766     Set database, storage, message configuration
 767     Init database with admin/admin user password
 768     """
 769     global ro_server, vim_admin_thread
 770     # global vim_threads
 771     cherrypy.log.error("Starting osm_ng_ro")
 772     # update general cherrypy configuration
 773     update_dict = {}
 774     engine_config = cherrypy.tree.apps["/ro"].config
 775
 776     for k, v in environ.items():
 777         if not k.startswith("OSMRO_"):
 778             continue
 779
 780         k1, _, k2 = k[6:].lower().partition("_")
 781
 782         if not k2:
 783             continue
 784
 785         try:
 786             if k1 in ("server", "test", "auth", "log"):
 787                 # update [global] configuration
 788                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 789             elif k1 == "static":
 790                 # update [/static] configuration
 791                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 792             elif k1 == "tools":
 793                 # update [/] configuration
 794                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 795             elif k1 in ("message", "database", "storage", "authentication"):
 796                 engine_config[k1][k2] = yaml.safe_load(v)
 797
 798         except Exception as e:
 799             raise RoException("Cannot load env '{}': {}".format(k, e))
 800
 801     if update_dict:
 802         cherrypy.config.update(update_dict)
 803         engine_config["global"].update(update_dict)
 804
 805     # logging cherrypy
 806     log_format_simple = (
 807         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 808     )
 809     log_formatter_simple = logging.Formatter(
 810         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 811     )
 812     logger_server = logging.getLogger("cherrypy.error")
 813     logger_access = logging.getLogger("cherrypy.access")
 814     logger_cherry = logging.getLogger("cherrypy")
 815     logger = logging.getLogger("ro")
 816
 817     if "log.file" in engine_config["global"]:
 818         file_handler = logging.handlers.RotatingFileHandler(
 819             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 820         )
 821         file_handler.setFormatter(log_formatter_simple)
 822         logger_cherry.addHandler(file_handler)
 823         logger.addHandler(file_handler)
 824
 825     # log always to standard output
 826     for format_, logger in {
 827         "ro.server %(filename)s:%(lineno)s": logger_server,
 828         "ro.access %(filename)s:%(lineno)s": logger_access,
 829         "%(name)s %(filename)s:%(lineno)s": logger,
 830     }.items():
 831         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 832         log_formatter_cherry = logging.Formatter(
 833             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 834         )
 835         str_handler = logging.StreamHandler()
 836         str_handler.setFormatter(log_formatter_cherry)
 837         logger.addHandler(str_handler)
 838
 839     if engine_config["global"].get("log.level"):
 840         logger_cherry.setLevel(engine_config["global"]["log.level"])
 841         logger.setLevel(engine_config["global"]["log.level"])
 842
 843     # logging other modules
 844     for k1, logname in {
 845         "message": "ro.msg",
 846         "database": "ro.db",
 847         "storage": "ro.fs",
 848     }.items():
 849         engine_config[k1]["logger_name"] = logname
 850         logger_module = logging.getLogger(logname)
 851
 852         if "logfile" in engine_config[k1]:
 853             file_handler = logging.handlers.RotatingFileHandler(
 854                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 855             )
 856             file_handler.setFormatter(log_formatter_simple)
 857             logger_module.addHandler(file_handler)
 858
 859         if "loglevel" in engine_config[k1]:
 860             logger_module.setLevel(engine_config[k1]["loglevel"])
 861     # TODO add more entries, e.g.: storage
 862
 863     engine_config["assignment"] = {}
 864     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 865     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 866     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 867     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 868
 869     # # start subscriptions thread:
 870     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 871     vim_admin_thread.start()
 872     # # Do not capture except SubscriptionException
 873
 874     # backend = engine_config["authentication"]["backend"]
 875     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 876     #                    .format(ro_version, ro_version_date, backend))
 877
 878
 879 def _stop_service():
 880     """
 881     Callback function called when cherrypy.engine stops
 882     TODO: Ending database connections.
 883     """
 884     global vim_admin_thread
 885
 886     # terminate vim_admin_thread
 887     if vim_admin_thread:
 888         vim_admin_thread.terminate()
 889
 890     vim_admin_thread = None
 891     cherrypy.tree.apps["/ro"].root.ns.stop()
 892     cherrypy.log.error("Stopping osm_ng_ro")
 893
 894
 895 def ro_main(config_file):
 896     global ro_server
 897
 898     ro_server = Server()
 899     cherrypy.engine.subscribe("start", _start_service)
 900     cherrypy.engine.subscribe("stop", _stop_service)
 901     cherrypy.quickstart(ro_server, "/ro", config_file)
 902
 903
 904 def usage():
 905     print(
 906         """Usage: {} [options]
 907         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 908         -h|--help: shows this help
 909         """.format(
 910             sys.argv[0]
 911         )
 912     )
 913     # --log-socket-host HOST: send logs to this host")
 914     # --log-socket-port PORT: send logs using this port (default: 9022)")
 915
 916
 917 if __name__ == "__main__":
 918     try:
 919         # load parameters and configuration
 920         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 921         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 922         config_file = None
 923
 924         for o, a in opts:
 925             if o in ("-h", "--help"):
 926                 usage()
 927                 sys.exit()
 928             elif o in ("-c", "--config"):
 929                 config_file = a
 930             else:
 931                 assert False, "Unhandled option"
 932
 933         if config_file:
 934             if not path.isfile(config_file):
 935                 print(
 936                     "configuration file '{}' that not exist".format(config_file),
 937                     file=sys.stderr,
 938                 )
 939                 exit(1)
 940         else:
 941             for config_file in (
 942                 path.dirname(__file__) + "/ro.cfg",
 943                 "./ro.cfg",
 944                 "/etc/osm/ro.cfg",
 945             ):
 946                 if path.isfile(config_file):
 947                     break
 948             else:
 949                 print(
 950                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 951                     file=sys.stderr,
 952                 )
 953                 exit(1)
 954
 955         ro_main(config_file)
 956     except KeyboardInterrupt:
 957         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 958     except getopt.GetoptError as e:
 959         print(str(e), file=sys.stderr)
 960         # usage()
 961         exit(1)