NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "deploy": {
  87                 "METHODS": ("GET",),
  88                 "ROLE_PERMISSION": "deploy:",
  89                 "<ID>": {
  90                     "METHODS": ("GET", "POST", "DELETE"),
  91                     "ROLE_PERMISSION": "deploy:id:",
  92                     "<ID>": {
  93                         "METHODS": ("GET",),
  94                         "ROLE_PERMISSION": "deploy:id:id:",
  95                         "cancel": {
  96                             "METHODS": ("POST",),
  97                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  98                         },
  99                     },
 100                 },
 101             },
 102             "recreate": {
 103                 "<ID>": {
 104                     "METHODS": ("POST"),
 105                     "ROLE_PERMISSION": "recreate:id:",
 106                     "<ID>": {
 107                         "METHODS": ("GET",),
 108                         "ROLE_PERMISSION": "recreate:id:id:",
 109                     },
 110                 },
 111             },
 112         }
 113     },
 114 }
 115
 116
 117 class RoException(Exception):
 118     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 119         Exception.__init__(self, message)
 120         self.http_code = http_code
 121
 122
 123 class AuthException(RoException):
 124     pass
 125
 126
 127 class Authenticator:
 128     def __init__(self, valid_url_methods, valid_query_string):
 129         self.valid_url_methods = valid_url_methods
 130         self.valid_query_string = valid_query_string
 131
 132     def authorize(self, *args, **kwargs):
 133         return {"token": "ok", "id": "ok"}
 134
 135     def new_token(self, token_info, indata, remote):
 136         return {"token": "ok", "id": "ok", "remote": remote}
 137
 138     def del_token(self, token_id):
 139         pass
 140
 141     def start(self, engine_config):
 142         pass
 143
 144
 145 class Server(object):
 146     instance = 0
 147     # to decode bytes to str
 148     reader = getreader("utf-8")
 149
 150     def __init__(self):
 151         self.instance += 1
 152         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 153         self.ns = Ns()
 154         self.map_operation = {
 155             "token:post": self.new_token,
 156             "token:id:delete": self.del_token,
 157             "deploy:get": self.ns.get_deploy,
 158             "deploy:id:get": self.ns.get_actions,
 159             "deploy:id:post": self.ns.deploy,
 160             "deploy:id:delete": self.ns.delete,
 161             "deploy:id:id:get": self.ns.status,
 162             "deploy:id:id:cancel:post": self.ns.cancel,
 163             "recreate:id:post": self.ns.recreate,
 164             "recreate:id:id:get": self.ns.recreate_status,
 165         }
 166
 167     def _format_in(self, kwargs):
 168         try:
 169             indata = None
 170
 171             if cherrypy.request.body.length:
 172                 error_text = "Invalid input format "
 173
 174                 if "Content-Type" in cherrypy.request.headers:
 175                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 176                         error_text = "Invalid json format "
 177                         indata = json.load(self.reader(cherrypy.request.body))
 178                         cherrypy.request.headers.pop("Content-File-MD5", None)
 179                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 180                         error_text = "Invalid yaml format "
 181                         indata = yaml.load(
 182                             cherrypy.request.body, Loader=yaml.SafeLoader
 183                         )
 184                         cherrypy.request.headers.pop("Content-File-MD5", None)
 185                     elif (
 186                         "application/binary" in cherrypy.request.headers["Content-Type"]
 187                         or "application/gzip"
 188                         in cherrypy.request.headers["Content-Type"]
 189                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 190                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 191                     ):
 192                         indata = cherrypy.request.body  # .read()
 193                     elif (
 194                         "multipart/form-data"
 195                         in cherrypy.request.headers["Content-Type"]
 196                     ):
 197                         if "descriptor_file" in kwargs:
 198                             filecontent = kwargs.pop("descriptor_file")
 199
 200                             if not filecontent.file:
 201                                 raise RoException(
 202                                     "empty file or content", HTTPStatus.BAD_REQUEST
 203                                 )
 204
 205                             indata = filecontent.file  # .read()
 206
 207                             if filecontent.content_type.value:
 208                                 cherrypy.request.headers[
 209                                     "Content-Type"
 210                                 ] = filecontent.content_type.value
 211                     else:
 212                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 213                         #                          "Only 'Content-Type' of type 'application/json' or
 214                         # 'application/yaml' for input format are available")
 215                         error_text = "Invalid yaml format "
 216                         indata = yaml.load(
 217                             cherrypy.request.body, Loader=yaml.SafeLoader
 218                         )
 219                         cherrypy.request.headers.pop("Content-File-MD5", None)
 220                 else:
 221                     error_text = "Invalid yaml format "
 222                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 223                     cherrypy.request.headers.pop("Content-File-MD5", None)
 224
 225             if not indata:
 226                 indata = {}
 227
 228             format_yaml = False
 229             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 230                 format_yaml = True
 231
 232             for k, v in kwargs.items():
 233                 if isinstance(v, str):
 234                     if v == "":
 235                         kwargs[k] = None
 236                     elif format_yaml:
 237                         try:
 238                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 239                         except Exception:
 240                             pass
 241                     elif (
 242                         k.endswith(".gt")
 243                         or k.endswith(".lt")
 244                         or k.endswith(".gte")
 245                         or k.endswith(".lte")
 246                     ):
 247                         try:
 248                             kwargs[k] = int(v)
 249                         except Exception:
 250                             try:
 251                                 kwargs[k] = float(v)
 252                             except Exception:
 253                                 pass
 254                     elif v.find(",") > 0:
 255                         kwargs[k] = v.split(",")
 256                 elif isinstance(v, (list, tuple)):
 257                     for index in range(0, len(v)):
 258                         if v[index] == "":
 259                             v[index] = None
 260                         elif format_yaml:
 261                             try:
 262                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 263                             except Exception:
 264                                 pass
 265
 266             return indata
 267         except (ValueError, yaml.YAMLError) as exc:
 268             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 269         except KeyError as exc:
 270             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 271         except Exception as exc:
 272             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 273
 274     @staticmethod
 275     def _format_out(data, token_info=None, _format=None):
 276         """
 277         return string of dictionary data according to requested json, yaml, xml. By default json
 278         :param data: response to be sent. Can be a dict, text or file
 279         :param token_info: Contains among other username and project
 280         :param _format: The format to be set as Content-Type if data is a file
 281         :return: None
 282         """
 283         accept = cherrypy.request.headers.get("Accept")
 284
 285         if data is None:
 286             if accept and "text/html" in accept:
 287                 return html.format(
 288                     data, cherrypy.request, cherrypy.response, token_info
 289                 )
 290
 291             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 292             return
 293         elif hasattr(data, "read"):  # file object
 294             if _format:
 295                 cherrypy.response.headers["Content-Type"] = _format
 296             elif "b" in data.mode:  # binariy asssumig zip
 297                 cherrypy.response.headers["Content-Type"] = "application/zip"
 298             else:
 299                 cherrypy.response.headers["Content-Type"] = "text/plain"
 300
 301             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 302             return data
 303
 304         if accept:
 305             if "application/json" in accept:
 306                 cherrypy.response.headers[
 307                     "Content-Type"
 308                 ] = "application/json; charset=utf-8"
 309                 a = json.dumps(data, indent=4) + "\n"
 310
 311                 return a.encode("utf8")
 312             elif "text/html" in accept:
 313                 return html.format(
 314                     data, cherrypy.request, cherrypy.response, token_info
 315                 )
 316             elif (
 317                 "application/yaml" in accept
 318                 or "*/*" in accept
 319                 or "text/plain" in accept
 320             ):
 321                 pass
 322             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 323             elif cherrypy.response.status >= 400:
 324                 raise cherrypy.HTTPError(
 325                     HTTPStatus.NOT_ACCEPTABLE.value,
 326                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 327                     "for output format are available",
 328                 )
 329
 330         cherrypy.response.headers["Content-Type"] = "application/yaml"
 331
 332         return yaml.safe_dump(
 333             data,
 334             explicit_start=True,
 335             indent=4,
 336             default_flow_style=False,
 337             tags=False,
 338             encoding="utf-8",
 339             allow_unicode=True,
 340         )  # , canonical=True, default_style='"'
 341
 342     @cherrypy.expose
 343     def index(self, *args, **kwargs):
 344         token_info = None
 345
 346         try:
 347             if cherrypy.request.method == "GET":
 348                 token_info = self.authenticator.authorize()
 349                 outdata = token_info  # Home page
 350             else:
 351                 raise cherrypy.HTTPError(
 352                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 353                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 354                 )
 355
 356             return self._format_out(outdata, token_info)
 357         except (NsException, AuthException) as e:
 358             # cherrypy.log("index Exception {}".format(e))
 359             cherrypy.response.status = e.http_code.value
 360
 361             return self._format_out("Welcome to OSM!", token_info)
 362
 363     @cherrypy.expose
 364     def version(self, *args, **kwargs):
 365         # TODO consider to remove and provide version using the static version file
 366         try:
 367             if cherrypy.request.method != "GET":
 368                 raise RoException(
 369                     "Only method GET is allowed",
 370                     HTTPStatus.METHOD_NOT_ALLOWED,
 371                 )
 372             elif args or kwargs:
 373                 raise RoException(
 374                     "Invalid URL or query string for version",
 375                     HTTPStatus.METHOD_NOT_ALLOWED,
 376                 )
 377
 378             # TODO include version of other modules, pick up from some kafka admin message
 379             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 380
 381             return self._format_out(osm_ng_ro_version)
 382         except RoException as e:
 383             cherrypy.response.status = e.http_code.value
 384             problem_details = {
 385                 "code": e.http_code.name,
 386                 "status": e.http_code.value,
 387                 "detail": str(e),
 388             }
 389
 390             return self._format_out(problem_details, None)
 391
 392     def new_token(self, engine_session, indata, *args, **kwargs):
 393         token_info = None
 394
 395         try:
 396             token_info = self.authenticator.authorize()
 397         except Exception:
 398             token_info = None
 399
 400         if kwargs:
 401             indata.update(kwargs)
 402
 403         # This is needed to log the user when authentication fails
 404         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 405         token_info = self.authenticator.new_token(
 406             token_info, indata, cherrypy.request.remote
 407         )
 408         cherrypy.session["Authorization"] = token_info["id"]
 409         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 410         # for logging
 411
 412         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 413         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 414
 415         return token_info, token_info["id"], True
 416
 417     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 418         token_id = _id
 419
 420         if not token_id and "id" in kwargs:
 421             token_id = kwargs["id"]
 422         elif not token_id:
 423             token_info = self.authenticator.authorize()
 424             # for logging
 425             token_id = token_info["id"]
 426
 427         self.authenticator.del_token(token_id)
 428         token_info = None
 429         cherrypy.session["Authorization"] = "logout"
 430         # cherrypy.response.cookie["Authorization"] = token_id
 431         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 432
 433         return None, None, True
 434
 435     @cherrypy.expose
 436     def test(self, *args, **kwargs):
 437         if not cherrypy.config.get("server.enable_test") or (
 438             isinstance(cherrypy.config["server.enable_test"], str)
 439             and cherrypy.config["server.enable_test"].lower() == "false"
 440         ):
 441             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 442
 443             return "test URL is disabled"
 444
 445         thread_info = None
 446
 447         if args and args[0] == "help":
 448             return (
 449                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 450                 "sleep/<time>\nmessage/topic\n</pre></html>"
 451             )
 452         elif args and args[0] == "init":
 453             try:
 454                 # self.ns.load_dbase(cherrypy.request.app.config)
 455                 self.ns.create_admin()
 456
 457                 return "Done. User 'admin', password 'admin' created"
 458             except Exception:
 459                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 460
 461                 return self._format_out("Database already initialized")
 462         elif args and args[0] == "file":
 463             return cherrypy.lib.static.serve_file(
 464                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 465                 "text/plain",
 466                 "attachment",
 467             )
 468         elif args and args[0] == "file2":
 469             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 470             f = open(f_path, "r")
 471             cherrypy.response.headers["Content-type"] = "text/plain"
 472             return f
 473
 474         elif len(args) == 2 and args[0] == "db-clear":
 475             deleted_info = self.ns.db.del_list(args[1], kwargs)
 476             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 477         elif len(args) and args[0] == "fs-clear":
 478             if len(args) >= 2:
 479                 folders = (args[1],)
 480             else:
 481                 folders = self.ns.fs.dir_ls(".")
 482
 483             for folder in folders:
 484                 self.ns.fs.file_delete(folder)
 485
 486             return ",".join(folders) + " folders deleted\n"
 487         elif args and args[0] == "login":
 488             if not cherrypy.request.headers.get("Authorization"):
 489                 cherrypy.response.headers[
 490                     "WWW-Authenticate"
 491                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 492                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 493         elif args and args[0] == "login2":
 494             if not cherrypy.request.headers.get("Authorization"):
 495                 cherrypy.response.headers[
 496                     "WWW-Authenticate"
 497                 ] = 'Bearer realm="Access to OSM site"'
 498                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 499         elif args and args[0] == "sleep":
 500             sleep_time = 5
 501
 502             try:
 503                 sleep_time = int(args[1])
 504             except Exception:
 505                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 506                 return self._format_out("Database already initialized")
 507
 508             thread_info = cherrypy.thread_data
 509             print(thread_info)
 510             time.sleep(sleep_time)
 511             # thread_info
 512         elif len(args) >= 2 and args[0] == "message":
 513             main_topic = args[1]
 514             return_text = "<html><pre>{} ->\n".format(main_topic)
 515
 516             try:
 517                 if cherrypy.request.method == "POST":
 518                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 519                     for k, v in to_send.items():
 520                         self.ns.msg.write(main_topic, k, v)
 521                         return_text += "  {}: {}\n".format(k, v)
 522                 elif cherrypy.request.method == "GET":
 523                     for k, v in kwargs.items():
 524                         self.ns.msg.write(
 525                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 526                         )
 527                         return_text += "  {}: {}\n".format(
 528                             k, yaml.load(v, Loader=yaml.SafeLoader)
 529                         )
 530             except Exception as e:
 531                 return_text += "Error: " + str(e)
 532
 533             return_text += "</pre></html>\n"
 534
 535             return return_text
 536
 537         return_text = (
 538             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 539             + "  kwargs: {}\n".format(kwargs)
 540             + "  headers: {}\n".format(cherrypy.request.headers)
 541             + "  path_info: {}\n".format(cherrypy.request.path_info)
 542             + "  query_string: {}\n".format(cherrypy.request.query_string)
 543             + "  session: {}\n".format(cherrypy.session)
 544             + "  cookie: {}\n".format(cherrypy.request.cookie)
 545             + "  method: {}\n".format(cherrypy.request.method)
 546             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 547             + "  body:\n"
 548         )
 549         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 550
 551         if cherrypy.request.body.length:
 552             return_text += "    content: {}\n".format(
 553                 str(
 554                     cherrypy.request.body.read(
 555                         int(cherrypy.request.headers.get("Content-Length", 0))
 556                     )
 557                 )
 558             )
 559
 560         if thread_info:
 561             return_text += "thread: {}\n".format(thread_info)
 562
 563         return_text += "</pre></html>"
 564
 565         return return_text
 566
 567     @staticmethod
 568     def _check_valid_url_method(method, *args):
 569         if len(args) < 3:
 570             raise RoException(
 571                 "URL must contain at least 'main_topic/version/topic'",
 572                 HTTPStatus.METHOD_NOT_ALLOWED,
 573             )
 574
 575         reference = valid_url_methods
 576         for arg in args:
 577             if arg is None:
 578                 break
 579
 580             if not isinstance(reference, dict):
 581                 raise RoException(
 582                     "URL contains unexpected extra items '{}'".format(arg),
 583                     HTTPStatus.METHOD_NOT_ALLOWED,
 584                 )
 585
 586             if arg in reference:
 587                 reference = reference[arg]
 588             elif "<ID>" in reference:
 589                 reference = reference["<ID>"]
 590             elif "*" in reference:
 591                 # reference = reference["*"]
 592                 break
 593             else:
 594                 raise RoException(
 595                     "Unexpected URL item {}".format(arg),
 596                     HTTPStatus.METHOD_NOT_ALLOWED,
 597                 )
 598
 599         if "TODO" in reference and method in reference["TODO"]:
 600             raise RoException(
 601                 "Method {} not supported yet for this URL".format(method),
 602                 HTTPStatus.NOT_IMPLEMENTED,
 603             )
 604         elif "METHODS" not in reference or method not in reference["METHODS"]:
 605             raise RoException(
 606                 "Method {} not supported for this URL".format(method),
 607                 HTTPStatus.METHOD_NOT_ALLOWED,
 608             )
 609
 610         return reference["ROLE_PERMISSION"] + method.lower()
 611
 612     @staticmethod
 613     def _set_location_header(main_topic, version, topic, id):
 614         """
 615         Insert response header Location with the URL of created item base on URL params
 616         :param main_topic:
 617         :param version:
 618         :param topic:
 619         :param id:
 620         :return: None
 621         """
 622         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 623         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 624             main_topic, version, topic, id
 625         )
 626
 627         return
 628
 629     @cherrypy.expose
 630     def default(
 631         self,
 632         main_topic=None,
 633         version=None,
 634         topic=None,
 635         _id=None,
 636         _id2=None,
 637         *args,
 638         **kwargs,
 639     ):
 640         token_info = None
 641         outdata = None
 642         _format = None
 643         method = "DONE"
 644         rollback = []
 645         engine_session = None
 646
 647         try:
 648             if not main_topic or not version or not topic:
 649                 raise RoException(
 650                     "URL must contain at least 'main_topic/version/topic'",
 651                     HTTPStatus.METHOD_NOT_ALLOWED,
 652                 )
 653
 654             if main_topic not in (
 655                 "admin",
 656                 "ns",
 657             ):
 658                 raise RoException(
 659                     "URL main_topic '{}' not supported".format(main_topic),
 660                     HTTPStatus.METHOD_NOT_ALLOWED,
 661                 )
 662
 663             if version != "v1":
 664                 raise RoException(
 665                     "URL version '{}' not supported".format(version),
 666                     HTTPStatus.METHOD_NOT_ALLOWED,
 667                 )
 668
 669             if (
 670                 kwargs
 671                 and "METHOD" in kwargs
 672                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 673             ):
 674                 method = kwargs.pop("METHOD")
 675             else:
 676                 method = cherrypy.request.method
 677
 678             role_permission = self._check_valid_url_method(
 679                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 680             )
 681             # skip token validation if requesting a token
 682             indata = self._format_in(kwargs)
 683
 684             if main_topic != "admin" or topic != "tokens":
 685                 token_info = self.authenticator.authorize(role_permission, _id)
 686
 687             outdata, created_id, done = self.map_operation[role_permission](
 688                 engine_session, indata, version, _id, _id2, *args, *kwargs
 689             )
 690
 691             if created_id:
 692                 self._set_location_header(main_topic, version, topic, _id)
 693
 694             cherrypy.response.status = (
 695                 HTTPStatus.ACCEPTED.value
 696                 if not done
 697                 else HTTPStatus.OK.value
 698                 if outdata is not None
 699                 else HTTPStatus.NO_CONTENT.value
 700             )
 701
 702             return self._format_out(outdata, token_info, _format)
 703         except Exception as e:
 704             if isinstance(
 705                 e,
 706                 (
 707                     RoException,
 708                     NsException,
 709                     DbException,
 710                     FsException,
 711                     MsgException,
 712                     AuthException,
 713                     ValidationError,
 714                 ),
 715             ):
 716                 http_code_value = cherrypy.response.status = e.http_code.value
 717                 http_code_name = e.http_code.name
 718                 cherrypy.log("Exception {}".format(e))
 719             else:
 720                 http_code_value = (
 721                     cherrypy.response.status
 722                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 723                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 724                 http_code_name = HTTPStatus.BAD_REQUEST.name
 725
 726             if hasattr(outdata, "close"):  # is an open file
 727                 outdata.close()
 728
 729             error_text = str(e)
 730             rollback.reverse()
 731
 732             for rollback_item in rollback:
 733                 try:
 734                     if rollback_item.get("operation") == "set":
 735                         self.ns.db.set_one(
 736                             rollback_item["topic"],
 737                             {"_id": rollback_item["_id"]},
 738                             rollback_item["content"],
 739                             fail_on_empty=False,
 740                         )
 741                     else:
 742                         self.ns.db.del_one(
 743                             rollback_item["topic"],
 744                             {"_id": rollback_item["_id"]},
 745                             fail_on_empty=False,
 746                         )
 747                 except Exception as e2:
 748                     rollback_error_text = "Rollback Exception {}: {}".format(
 749                         rollback_item, e2
 750                     )
 751                     cherrypy.log(rollback_error_text)
 752                     error_text += ". " + rollback_error_text
 753
 754             # if isinstance(e, MsgException):
 755             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 756             #         engine_topic[:-1], method, error_text)
 757             problem_details = {
 758                 "code": http_code_name,
 759                 "status": http_code_value,
 760                 "detail": error_text,
 761             }
 762
 763             return self._format_out(problem_details, token_info)
 764             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 765         finally:
 766             if token_info:
 767                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 768                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 769                         if outdata.get(logging_id):
 770                             cherrypy.request.login += ";{}={}".format(
 771                                 logging_id, outdata[logging_id][:36]
 772                             )
 773
 774
 775 def _start_service():
 776     """
 777     Callback function called when cherrypy.engine starts
 778     Override configuration with env variables
 779     Set database, storage, message configuration
 780     Init database with admin/admin user password
 781     """
 782     global ro_server, vim_admin_thread
 783     # global vim_threads
 784     cherrypy.log.error("Starting osm_ng_ro")
 785     # update general cherrypy configuration
 786     update_dict = {}
 787     engine_config = cherrypy.tree.apps["/ro"].config
 788
 789     for k, v in environ.items():
 790         if not k.startswith("OSMRO_"):
 791             continue
 792
 793         k1, _, k2 = k[6:].lower().partition("_")
 794
 795         if not k2:
 796             continue
 797
 798         try:
 799             if k1 in ("server", "test", "auth", "log"):
 800                 # update [global] configuration
 801                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 802             elif k1 == "static":
 803                 # update [/static] configuration
 804                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 805             elif k1 == "tools":
 806                 # update [/] configuration
 807                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 808             elif k1 in ("message", "database", "storage", "authentication"):
 809                 engine_config[k1][k2] = yaml.safe_load(v)
 810
 811         except Exception as e:
 812             raise RoException("Cannot load env '{}': {}".format(k, e))
 813
 814     if update_dict:
 815         cherrypy.config.update(update_dict)
 816         engine_config["global"].update(update_dict)
 817
 818     # logging cherrypy
 819     log_format_simple = (
 820         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 821     )
 822     log_formatter_simple = logging.Formatter(
 823         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 824     )
 825     logger_server = logging.getLogger("cherrypy.error")
 826     logger_access = logging.getLogger("cherrypy.access")
 827     logger_cherry = logging.getLogger("cherrypy")
 828     logger = logging.getLogger("ro")
 829
 830     if "log.file" in engine_config["global"]:
 831         file_handler = logging.handlers.RotatingFileHandler(
 832             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 833         )
 834         file_handler.setFormatter(log_formatter_simple)
 835         logger_cherry.addHandler(file_handler)
 836         logger.addHandler(file_handler)
 837
 838     # log always to standard output
 839     for format_, logger in {
 840         "ro.server %(filename)s:%(lineno)s": logger_server,
 841         "ro.access %(filename)s:%(lineno)s": logger_access,
 842         "%(name)s %(filename)s:%(lineno)s": logger,
 843     }.items():
 844         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 845         log_formatter_cherry = logging.Formatter(
 846             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 847         )
 848         str_handler = logging.StreamHandler()
 849         str_handler.setFormatter(log_formatter_cherry)
 850         logger.addHandler(str_handler)
 851
 852     if engine_config["global"].get("log.level"):
 853         logger_cherry.setLevel(engine_config["global"]["log.level"])
 854         logger.setLevel(engine_config["global"]["log.level"])
 855
 856     # logging other modules
 857     for k1, logname in {
 858         "message": "ro.msg",
 859         "database": "ro.db",
 860         "storage": "ro.fs",
 861     }.items():
 862         engine_config[k1]["logger_name"] = logname
 863         logger_module = logging.getLogger(logname)
 864
 865         if "logfile" in engine_config[k1]:
 866             file_handler = logging.handlers.RotatingFileHandler(
 867                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 868             )
 869             file_handler.setFormatter(log_formatter_simple)
 870             logger_module.addHandler(file_handler)
 871
 872         if "loglevel" in engine_config[k1]:
 873             logger_module.setLevel(engine_config[k1]["loglevel"])
 874     # TODO add more entries, e.g.: storage
 875
 876     engine_config["assignment"] = {}
 877     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 878     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 879     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 880     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 881
 882     # # start subscriptions thread:
 883     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 884     vim_admin_thread.start()
 885     # # Do not capture except SubscriptionException
 886
 887     # backend = engine_config["authentication"]["backend"]
 888     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 889     #                    .format(ro_version, ro_version_date, backend))
 890
 891
 892 def _stop_service():
 893     """
 894     Callback function called when cherrypy.engine stops
 895     TODO: Ending database connections.
 896     """
 897     global vim_admin_thread
 898
 899     # terminate vim_admin_thread
 900     if vim_admin_thread:
 901         vim_admin_thread.terminate()
 902
 903     vim_admin_thread = None
 904     cherrypy.tree.apps["/ro"].root.ns.stop()
 905     cherrypy.log.error("Stopping osm_ng_ro")
 906
 907
 908 def ro_main(config_file):
 909     global ro_server
 910
 911     ro_server = Server()
 912     cherrypy.engine.subscribe("start", _start_service)
 913     cherrypy.engine.subscribe("stop", _stop_service)
 914     cherrypy.quickstart(ro_server, "/ro", config_file)
 915
 916
 917 def usage():
 918     print(
 919         """Usage: {} [options]
 920         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 921         -h|--help: shows this help
 922         """.format(
 923             sys.argv[0]
 924         )
 925     )
 926     # --log-socket-host HOST: send logs to this host")
 927     # --log-socket-port PORT: send logs using this port (default: 9022)")
 928
 929
 930 if __name__ == "__main__":
 931     try:
 932         # load parameters and configuration
 933         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 934         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 935         config_file = None
 936
 937         for o, a in opts:
 938             if o in ("-h", "--help"):
 939                 usage()
 940                 sys.exit()
 941             elif o in ("-c", "--config"):
 942                 config_file = a
 943             else:
 944                 assert False, "Unhandled option"
 945
 946         if config_file:
 947             if not path.isfile(config_file):
 948                 print(
 949                     "configuration file '{}' that not exist".format(config_file),
 950                     file=sys.stderr,
 951                 )
 952                 exit(1)
 953         else:
 954             for config_file in (
 955                 path.dirname(__file__) + "/ro.cfg",
 956                 "./ro.cfg",
 957                 "/etc/osm/ro.cfg",
 958             ):
 959                 if path.isfile(config_file):
 960                     break
 961             else:
 962                 print(
 963                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 964                     file=sys.stderr,
 965                 )
 966                 exit(1)
 967
 968         ro_main(config_file)
 969     except KeyboardInterrupt:
 970         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 971     except getopt.GetoptError as e:
 972         print(str(e), file=sys.stderr)
 973         # usage()
 974         exit(1)