NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21 from codecs import getreader
  22 import getopt
  23 from http import HTTPStatus
  24 import json
  25 import logging
  26 import logging.handlers
  27 from os import environ, path
  28 import sys
  29 import time
  30
  31 import cherrypy
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  36 import osm_ng_ro.html_out as html
  37 from osm_ng_ro.ns import Ns, NsException
  38 from osm_ng_ro.validation import ValidationError
  39 from osm_ng_ro.vim_admin import VimAdminThread
  40 import yaml
  41
  42 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  43
  44 __version__ = "0.1."  # file version, not NBI version
  45 version_date = "May 2020"
  46
  47 database_version = "1.2"
  48 auth_database_version = "1.0"
  49 ro_server = None  # instance of Server class
  50 vim_admin_thread = None  # instance of VimAdminThread class
  51
  52 # vim_threads = None  # instance of VimThread class
  53
  54 """
  55 RO North Bound Interface
  56 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  57         /ns/v1/deploy                                           O
  58             /<nsrs_id>                                          O       O               O
  59                 /<action_id>                                    O
  60                     /cancel                                             O
  61
  62 """
  63
  64 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  65 # ^ Contains possible administrative query string words:
  66 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  67 #           (not owned by my session project).
  68 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  69 #     FORCE=True(by default)|False: Force edition/deletion operations
  70 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  71
  72 valid_url_methods = {
  73     # contains allowed URL and methods, and the role_permission name
  74     "admin": {
  75         "v1": {
  76             "tokens": {
  77                 "METHODS": ("POST",),
  78                 "ROLE_PERMISSION": "tokens:",
  79                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  80             },
  81         }
  82     },
  83     "ns": {
  84         "v1": {
  85             "deploy": {
  86                 "METHODS": ("GET",),
  87                 "ROLE_PERMISSION": "deploy:",
  88                 "<ID>": {
  89                     "METHODS": ("GET", "POST", "DELETE"),
  90                     "ROLE_PERMISSION": "deploy:id:",
  91                     "<ID>": {
  92                         "METHODS": ("GET",),
  93                         "ROLE_PERMISSION": "deploy:id:id:",
  94                         "cancel": {
  95                             "METHODS": ("POST",),
  96                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  97                         },
  98                     },
  99                 },
 100             },
 101         }
 102     },
 103 }
 104
 105
 106 class RoException(Exception):
 107     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 108         Exception.__init__(self, message)
 109         self.http_code = http_code
 110
 111
 112 class AuthException(RoException):
 113     pass
 114
 115
 116 class Authenticator:
 117     def __init__(self, valid_url_methods, valid_query_string):
 118         self.valid_url_methods = valid_url_methods
 119         self.valid_query_string = valid_query_string
 120
 121     def authorize(self, *args, **kwargs):
 122         return {"token": "ok", "id": "ok"}
 123
 124     def new_token(self, token_info, indata, remote):
 125         return {"token": "ok", "id": "ok", "remote": remote}
 126
 127     def del_token(self, token_id):
 128         pass
 129
 130     def start(self, engine_config):
 131         pass
 132
 133
 134 class Server(object):
 135     instance = 0
 136     # to decode bytes to str
 137     reader = getreader("utf-8")
 138
 139     def __init__(self):
 140         self.instance += 1
 141         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 142         self.ns = Ns()
 143         self.map_operation = {
 144             "token:post": self.new_token,
 145             "token:id:delete": self.del_token,
 146             "deploy:get": self.ns.get_deploy,
 147             "deploy:id:get": self.ns.get_actions,
 148             "deploy:id:post": self.ns.deploy,
 149             "deploy:id:delete": self.ns.delete,
 150             "deploy:id:id:get": self.ns.status,
 151             "deploy:id:id:cancel:post": self.ns.cancel,
 152         }
 153
 154     def _format_in(self, kwargs):
 155         try:
 156             indata = None
 157
 158             if cherrypy.request.body.length:
 159                 error_text = "Invalid input format "
 160
 161                 if "Content-Type" in cherrypy.request.headers:
 162                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 163                         error_text = "Invalid json format "
 164                         indata = json.load(self.reader(cherrypy.request.body))
 165                         cherrypy.request.headers.pop("Content-File-MD5", None)
 166                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 167                         error_text = "Invalid yaml format "
 168                         indata = yaml.safe_load(cherrypy.request.body)
 169                         cherrypy.request.headers.pop("Content-File-MD5", None)
 170                     elif (
 171                         "application/binary" in cherrypy.request.headers["Content-Type"]
 172                         or "application/gzip"
 173                         in cherrypy.request.headers["Content-Type"]
 174                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 175                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 176                     ):
 177                         indata = cherrypy.request.body  # .read()
 178                     elif (
 179                         "multipart/form-data"
 180                         in cherrypy.request.headers["Content-Type"]
 181                     ):
 182                         if "descriptor_file" in kwargs:
 183                             filecontent = kwargs.pop("descriptor_file")
 184
 185                             if not filecontent.file:
 186                                 raise RoException(
 187                                     "empty file or content", HTTPStatus.BAD_REQUEST
 188                                 )
 189
 190                             indata = filecontent.file  # .read()
 191
 192                             if filecontent.content_type.value:
 193                                 cherrypy.request.headers[
 194                                     "Content-Type"
 195                                 ] = filecontent.content_type.value
 196                     else:
 197                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 198                         #                          "Only 'Content-Type' of type 'application/json' or
 199                         # 'application/yaml' for input format are available")
 200                         error_text = "Invalid yaml format "
 201                         indata = yaml.safe_load(cherrypy.request.body)
 202                         cherrypy.request.headers.pop("Content-File-MD5", None)
 203                 else:
 204                     error_text = "Invalid yaml format "
 205                     indata = yaml.safe_load(cherrypy.request.body)
 206                     cherrypy.request.headers.pop("Content-File-MD5", None)
 207
 208             if not indata:
 209                 indata = {}
 210
 211             format_yaml = False
 212             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 213                 format_yaml = True
 214
 215             for k, v in kwargs.items():
 216                 if isinstance(v, str):
 217                     if v == "":
 218                         kwargs[k] = None
 219                     elif format_yaml:
 220                         try:
 221                             kwargs[k] = yaml.safe_load(v)
 222                         except Exception as yaml_error:
 223                             logging.exception(
 224                                 f"{yaml_error} occured while parsing the yaml"
 225                             )
 226                     elif (
 227                         k.endswith(".gt")
 228                         or k.endswith(".lt")
 229                         or k.endswith(".gte")
 230                         or k.endswith(".lte")
 231                     ):
 232                         try:
 233                             kwargs[k] = int(v)
 234                         except Exception:
 235                             try:
 236                                 kwargs[k] = float(v)
 237                             except Exception as keyword_error:
 238                                 logging.exception(
 239                                     f"{keyword_error} occured while getting the keyword arguments"
 240                                 )
 241                     elif v.find(",") > 0:
 242                         kwargs[k] = v.split(",")
 243                 elif isinstance(v, (list, tuple)):
 244                     for index in range(0, len(v)):
 245                         if v[index] == "":
 246                             v[index] = None
 247                         elif format_yaml:
 248                             try:
 249                                 v[index] = yaml.safe_load(v[index])
 250                             except Exception as error:
 251                                 logging.exception(
 252                                     f"{error} occured while parsing the yaml"
 253                                 )
 254
 255             return indata
 256         except (ValueError, yaml.YAMLError) as exc:
 257             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 258         except KeyError as exc:
 259             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 260         except Exception as exc:
 261             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 262
 263     @staticmethod
 264     def _format_out(data, token_info=None, _format=None):
 265         """
 266         return string of dictionary data according to requested json, yaml, xml. By default json
 267         :param data: response to be sent. Can be a dict, text or file
 268         :param token_info: Contains among other username and project
 269         :param _format: The format to be set as Content-Type if data is a file
 270         :return: None
 271         """
 272         accept = cherrypy.request.headers.get("Accept")
 273
 274         if data is None:
 275             if accept and "text/html" in accept:
 276                 return html.format(
 277                     data, cherrypy.request, cherrypy.response, token_info
 278                 )
 279
 280             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 281             return
 282         elif hasattr(data, "read"):  # file object
 283             if _format:
 284                 cherrypy.response.headers["Content-Type"] = _format
 285             elif "b" in data.mode:  # binariy asssumig zip
 286                 cherrypy.response.headers["Content-Type"] = "application/zip"
 287             else:
 288                 cherrypy.response.headers["Content-Type"] = "text/plain"
 289
 290             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 291             return data
 292
 293         if accept:
 294             if "application/json" in accept:
 295                 cherrypy.response.headers[
 296                     "Content-Type"
 297                 ] = "application/json; charset=utf-8"
 298                 a = json.dumps(data, indent=4) + "\n"
 299
 300                 return a.encode("utf8")
 301             elif "text/html" in accept:
 302                 return html.format(
 303                     data, cherrypy.request, cherrypy.response, token_info
 304                 )
 305             elif (
 306                 "application/yaml" in accept
 307                 or "*/*" in accept
 308                 or "text/plain" in accept
 309             ):
 310                 pass
 311             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 312             elif cherrypy.response.status >= 400:
 313                 raise cherrypy.HTTPError(
 314                     HTTPStatus.NOT_ACCEPTABLE.value,
 315                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 316                     "for output format are available",
 317                 )
 318
 319         cherrypy.response.headers["Content-Type"] = "application/yaml"
 320
 321         return yaml.safe_dump(
 322             data,
 323             explicit_start=True,
 324             indent=4,
 325             default_flow_style=False,
 326             tags=False,
 327             encoding="utf-8",
 328             allow_unicode=True,
 329         )  # , canonical=True, default_style='"'
 330
 331     @cherrypy.expose
 332     def index(self, *args, **kwargs):
 333         token_info = None
 334
 335         try:
 336             if cherrypy.request.method == "GET":
 337                 token_info = self.authenticator.authorize()
 338                 outdata = token_info  # Home page
 339             else:
 340                 raise cherrypy.HTTPError(
 341                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 342                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 343                 )
 344
 345             return self._format_out(outdata, token_info)
 346         except (NsException, AuthException) as e:
 347             # cherrypy.log("index Exception {}".format(e))
 348             cherrypy.response.status = e.http_code.value
 349
 350             return self._format_out("Welcome to OSM!", token_info)
 351
 352     @cherrypy.expose
 353     def version(self, *args, **kwargs):
 354         # TODO consider to remove and provide version using the static version file
 355         try:
 356             if cherrypy.request.method != "GET":
 357                 raise RoException(
 358                     "Only method GET is allowed",
 359                     HTTPStatus.METHOD_NOT_ALLOWED,
 360                 )
 361             elif args or kwargs:
 362                 raise RoException(
 363                     "Invalid URL or query string for version",
 364                     HTTPStatus.METHOD_NOT_ALLOWED,
 365                 )
 366
 367             # TODO include version of other modules, pick up from some kafka admin message
 368             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 369
 370             return self._format_out(osm_ng_ro_version)
 371         except RoException as e:
 372             cherrypy.response.status = e.http_code.value
 373             problem_details = {
 374                 "code": e.http_code.name,
 375                 "status": e.http_code.value,
 376                 "detail": str(e),
 377             }
 378
 379             return self._format_out(problem_details, None)
 380
 381     def new_token(self, engine_session, indata, *args, **kwargs):
 382         token_info = None
 383
 384         try:
 385             token_info = self.authenticator.authorize()
 386         except Exception:
 387             token_info = None
 388
 389         if kwargs:
 390             indata.update(kwargs)
 391
 392         # This is needed to log the user when authentication fails
 393         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 394         token_info = self.authenticator.new_token(
 395             token_info, indata, cherrypy.request.remote
 396         )
 397         cherrypy.session["Authorization"] = token_info["id"]
 398         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 399         # for logging
 400
 401         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 402         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 403
 404         return token_info, token_info["id"], True
 405
 406     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 407         token_id = _id
 408
 409         if not token_id and "id" in kwargs:
 410             token_id = kwargs["id"]
 411         elif not token_id:
 412             token_info = self.authenticator.authorize()
 413             # for logging
 414             token_id = token_info["id"]
 415
 416         self.authenticator.del_token(token_id)
 417         token_info = None
 418         cherrypy.session["Authorization"] = "logout"
 419         # cherrypy.response.cookie["Authorization"] = token_id
 420         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 421
 422         return None, None, True
 423
 424     @cherrypy.expose
 425     def test(self, *args, **kwargs):
 426         if not cherrypy.config.get("server.enable_test") or (
 427             isinstance(cherrypy.config["server.enable_test"], str)
 428             and cherrypy.config["server.enable_test"].lower() == "false"
 429         ):
 430             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 431
 432             return "test URL is disabled"
 433
 434         thread_info = None
 435
 436         if args and args[0] == "help":
 437             return (
 438                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 439                 "sleep/<time>\nmessage/topic\n</pre></html>"
 440             )
 441         elif args and args[0] == "init":
 442             try:
 443                 # self.ns.load_dbase(cherrypy.request.app.config)
 444                 self.ns.create_admin()
 445
 446                 return "Done. User 'admin', password 'admin' created"
 447             except Exception:
 448                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 449
 450                 return self._format_out("Database already initialized")
 451         elif args and args[0] == "file":
 452             return cherrypy.lib.static.serve_file(
 453                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 454                 "text/plain",
 455                 "attachment",
 456             )
 457         elif args and args[0] == "file2":
 458             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 459             f = open(f_path, "r")
 460             cherrypy.response.headers["Content-type"] = "text/plain"
 461             return f
 462
 463         elif len(args) == 2 and args[0] == "db-clear":
 464             deleted_info = self.ns.db.del_list(args[1], kwargs)
 465             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 466         elif len(args) and args[0] == "fs-clear":
 467             if len(args) >= 2:
 468                 folders = (args[1],)
 469             else:
 470                 folders = self.ns.fs.dir_ls(".")
 471
 472             for folder in folders:
 473                 self.ns.fs.file_delete(folder)
 474
 475             return ",".join(folders) + " folders deleted\n"
 476         elif args and args[0] == "login":
 477             if not cherrypy.request.headers.get("Authorization"):
 478                 cherrypy.response.headers[
 479                     "WWW-Authenticate"
 480                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 481                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 482         elif args and args[0] == "login2":
 483             if not cherrypy.request.headers.get("Authorization"):
 484                 cherrypy.response.headers[
 485                     "WWW-Authenticate"
 486                 ] = 'Bearer realm="Access to OSM site"'
 487                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 488         elif args and args[0] == "sleep":
 489             sleep_time = 5
 490
 491             try:
 492                 sleep_time = int(args[1])
 493             except Exception:
 494                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 495                 return self._format_out("Database already initialized")
 496
 497             thread_info = cherrypy.thread_data
 498             print(thread_info)
 499             time.sleep(sleep_time)
 500             # thread_info
 501         elif len(args) >= 2 and args[0] == "message":
 502             main_topic = args[1]
 503             return_text = "<html><pre>{} ->\n".format(main_topic)
 504
 505             try:
 506                 if cherrypy.request.method == "POST":
 507                     to_send = yaml.safe_load(cherrypy.request.body)
 508                     for k, v in to_send.items():
 509                         self.ns.msg.write(main_topic, k, v)
 510                         return_text += "  {}: {}\n".format(k, v)
 511                 elif cherrypy.request.method == "GET":
 512                     for k, v in kwargs.items():
 513                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 514                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 515             except Exception as e:
 516                 return_text += "Error: " + str(e)
 517
 518             return_text += "</pre></html>\n"
 519
 520             return return_text
 521
 522         return_text = (
 523             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 524             + "  kwargs: {}\n".format(kwargs)
 525             + "  headers: {}\n".format(cherrypy.request.headers)
 526             + "  path_info: {}\n".format(cherrypy.request.path_info)
 527             + "  query_string: {}\n".format(cherrypy.request.query_string)
 528             + "  session: {}\n".format(cherrypy.session)
 529             + "  cookie: {}\n".format(cherrypy.request.cookie)
 530             + "  method: {}\n".format(cherrypy.request.method)
 531             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 532             + "  body:\n"
 533         )
 534         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 535
 536         if cherrypy.request.body.length:
 537             return_text += "    content: {}\n".format(
 538                 str(
 539                     cherrypy.request.body.read(
 540                         int(cherrypy.request.headers.get("Content-Length", 0))
 541                     )
 542                 )
 543             )
 544
 545         if thread_info:
 546             return_text += "thread: {}\n".format(thread_info)
 547
 548         return_text += "</pre></html>"
 549
 550         return return_text
 551
 552     @staticmethod
 553     def _check_valid_url_method(method, *args):
 554         if len(args) < 3:
 555             raise RoException(
 556                 "URL must contain at least 'main_topic/version/topic'",
 557                 HTTPStatus.METHOD_NOT_ALLOWED,
 558             )
 559
 560         reference = valid_url_methods
 561         for arg in args:
 562             if arg is None:
 563                 break
 564
 565             if not isinstance(reference, dict):
 566                 raise RoException(
 567                     "URL contains unexpected extra items '{}'".format(arg),
 568                     HTTPStatus.METHOD_NOT_ALLOWED,
 569                 )
 570
 571             if arg in reference:
 572                 reference = reference[arg]
 573             elif "<ID>" in reference:
 574                 reference = reference["<ID>"]
 575             elif "*" in reference:
 576                 # reference = reference["*"]
 577                 break
 578             else:
 579                 raise RoException(
 580                     "Unexpected URL item {}".format(arg),
 581                     HTTPStatus.METHOD_NOT_ALLOWED,
 582                 )
 583
 584         if "TODO" in reference and method in reference["TODO"]:
 585             raise RoException(
 586                 "Method {} not supported yet for this URL".format(method),
 587                 HTTPStatus.NOT_IMPLEMENTED,
 588             )
 589         elif "METHODS" not in reference or method not in reference["METHODS"]:
 590             raise RoException(
 591                 "Method {} not supported for this URL".format(method),
 592                 HTTPStatus.METHOD_NOT_ALLOWED,
 593             )
 594
 595         return reference["ROLE_PERMISSION"] + method.lower()
 596
 597     @staticmethod
 598     def _set_location_header(main_topic, version, topic, id):
 599         """
 600         Insert response header Location with the URL of created item base on URL params
 601         :param main_topic:
 602         :param version:
 603         :param topic:
 604         :param id:
 605         :return: None
 606         """
 607         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 608         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 609             main_topic, version, topic, id
 610         )
 611
 612         return
 613
 614     @cherrypy.expose
 615     def default(
 616         self,
 617         main_topic=None,
 618         version=None,
 619         topic=None,
 620         _id=None,
 621         _id2=None,
 622         *args,
 623         **kwargs,
 624     ):
 625         token_info = None
 626         outdata = None
 627         _format = None
 628         method = "DONE"
 629         rollback = []
 630         engine_session = None
 631
 632         try:
 633             if not main_topic or not version or not topic:
 634                 raise RoException(
 635                     "URL must contain at least 'main_topic/version/topic'",
 636                     HTTPStatus.METHOD_NOT_ALLOWED,
 637                 )
 638
 639             if main_topic not in (
 640                 "admin",
 641                 "ns",
 642             ):
 643                 raise RoException(
 644                     "URL main_topic '{}' not supported".format(main_topic),
 645                     HTTPStatus.METHOD_NOT_ALLOWED,
 646                 )
 647
 648             if version != "v1":
 649                 raise RoException(
 650                     "URL version '{}' not supported".format(version),
 651                     HTTPStatus.METHOD_NOT_ALLOWED,
 652                 )
 653
 654             if (
 655                 kwargs
 656                 and "METHOD" in kwargs
 657                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 658             ):
 659                 method = kwargs.pop("METHOD")
 660             else:
 661                 method = cherrypy.request.method
 662
 663             role_permission = self._check_valid_url_method(
 664                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 665             )
 666             # skip token validation if requesting a token
 667             indata = self._format_in(kwargs)
 668
 669             if main_topic != "admin" or topic != "tokens":
 670                 token_info = self.authenticator.authorize(role_permission, _id)
 671
 672             outdata, created_id, done = self.map_operation[role_permission](
 673                 engine_session, indata, version, _id, _id2, *args, *kwargs
 674             )
 675
 676             if created_id:
 677                 self._set_location_header(main_topic, version, topic, _id)
 678
 679             cherrypy.response.status = (
 680                 HTTPStatus.ACCEPTED.value
 681                 if not done
 682                 else HTTPStatus.OK.value
 683                 if outdata is not None
 684                 else HTTPStatus.NO_CONTENT.value
 685             )
 686
 687             return self._format_out(outdata, token_info, _format)
 688         except Exception as e:
 689             if isinstance(
 690                 e,
 691                 (
 692                     RoException,
 693                     NsException,
 694                     DbException,
 695                     FsException,
 696                     MsgException,
 697                     AuthException,
 698                     ValidationError,
 699                 ),
 700             ):
 701                 http_code_value = cherrypy.response.status = e.http_code.value
 702                 http_code_name = e.http_code.name
 703                 cherrypy.log("Exception {}".format(e))
 704             else:
 705                 http_code_value = (
 706                     cherrypy.response.status
 707                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 708                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 709                 http_code_name = HTTPStatus.BAD_REQUEST.name
 710
 711             if hasattr(outdata, "close"):  # is an open file
 712                 outdata.close()
 713
 714             error_text = str(e)
 715             rollback.reverse()
 716
 717             for rollback_item in rollback:
 718                 try:
 719                     if rollback_item.get("operation") == "set":
 720                         self.ns.db.set_one(
 721                             rollback_item["topic"],
 722                             {"_id": rollback_item["_id"]},
 723                             rollback_item["content"],
 724                             fail_on_empty=False,
 725                         )
 726                     else:
 727                         self.ns.db.del_one(
 728                             rollback_item["topic"],
 729                             {"_id": rollback_item["_id"]},
 730                             fail_on_empty=False,
 731                         )
 732                 except Exception as e2:
 733                     rollback_error_text = "Rollback Exception {}: {}".format(
 734                         rollback_item, e2
 735                     )
 736                     cherrypy.log(rollback_error_text)
 737                     error_text += ". " + rollback_error_text
 738
 739             # if isinstance(e, MsgException):
 740             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 741             #         engine_topic[:-1], method, error_text)
 742             problem_details = {
 743                 "code": http_code_name,
 744                 "status": http_code_value,
 745                 "detail": error_text,
 746             }
 747
 748             return self._format_out(problem_details, token_info)
 749             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 750         finally:
 751             if token_info:
 752                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 753                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 754                         if outdata.get(logging_id):
 755                             cherrypy.request.login += ";{}={}".format(
 756                                 logging_id, outdata[logging_id][:36]
 757                             )
 758
 759
 760 def _start_service():
 761     """
 762     Callback function called when cherrypy.engine starts
 763     Override configuration with env variables
 764     Set database, storage, message configuration
 765     Init database with admin/admin user password
 766     """
 767     global ro_server, vim_admin_thread
 768     # global vim_threads
 769     cherrypy.log.error("Starting osm_ng_ro")
 770     # update general cherrypy configuration
 771     update_dict = {}
 772     engine_config = cherrypy.tree.apps["/ro"].config
 773
 774     for k, v in environ.items():
 775         if not k.startswith("OSMRO_"):
 776             continue
 777
 778         k1, _, k2 = k[6:].lower().partition("_")
 779
 780         if not k2:
 781             continue
 782
 783         try:
 784             if k1 in ("server", "test", "auth", "log"):
 785                 # update [global] configuration
 786                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 787             elif k1 == "static":
 788                 # update [/static] configuration
 789                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 790             elif k1 == "tools":
 791                 # update [/] configuration
 792                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 793             elif k1 in ("message", "database", "storage", "authentication", "period"):
 794                 engine_config[k1][k2] = yaml.safe_load(v)
 795
 796         except Exception as e:
 797             raise RoException("Cannot load env '{}': {}".format(k, e))
 798
 799     if update_dict:
 800         cherrypy.config.update(update_dict)
 801         engine_config["global"].update(update_dict)
 802
 803     # logging cherrypy
 804     log_format_simple = (
 805         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 806     )
 807     log_formatter_simple = logging.Formatter(
 808         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 809     )
 810     logger_server = logging.getLogger("cherrypy.error")
 811     logger_access = logging.getLogger("cherrypy.access")
 812     logger_cherry = logging.getLogger("cherrypy")
 813     logger = logging.getLogger("ro")
 814
 815     if "log.file" in engine_config["global"]:
 816         file_handler = logging.handlers.RotatingFileHandler(
 817             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 818         )
 819         file_handler.setFormatter(log_formatter_simple)
 820         logger_cherry.addHandler(file_handler)
 821         logger.addHandler(file_handler)
 822
 823     # log always to standard output
 824     for format_, logger in {
 825         "ro.server %(filename)s:%(lineno)s": logger_server,
 826         "ro.access %(filename)s:%(lineno)s": logger_access,
 827         "%(name)s %(filename)s:%(lineno)s": logger,
 828     }.items():
 829         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 830         log_formatter_cherry = logging.Formatter(
 831             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 832         )
 833         str_handler = logging.StreamHandler()
 834         str_handler.setFormatter(log_formatter_cherry)
 835         logger.addHandler(str_handler)
 836
 837     if engine_config["global"].get("log.level"):
 838         logger_cherry.setLevel(engine_config["global"]["log.level"])
 839         logger.setLevel(engine_config["global"]["log.level"])
 840
 841     # logging other modules
 842     for k1, logname in {
 843         "message": "ro.msg",
 844         "database": "ro.db",
 845         "storage": "ro.fs",
 846     }.items():
 847         engine_config[k1]["logger_name"] = logname
 848         logger_module = logging.getLogger(logname)
 849
 850         if "logfile" in engine_config[k1]:
 851             file_handler = logging.handlers.RotatingFileHandler(
 852                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 853             )
 854             file_handler.setFormatter(log_formatter_simple)
 855             logger_module.addHandler(file_handler)
 856
 857         if "loglevel" in engine_config[k1]:
 858             logger_module.setLevel(engine_config[k1]["loglevel"])
 859     # TODO add more entries, e.g.: storage
 860
 861     engine_config["assignment"] = {}
 862     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 863     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 864     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 865     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 866
 867     # # start subscriptions thread:
 868     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 869     vim_admin_thread.start()
 870     # # Do not capture except SubscriptionException
 871
 872     # backend = engine_config["authentication"]["backend"]
 873     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 874     #                    .format(ro_version, ro_version_date, backend))
 875
 876
 877 def _stop_service():
 878     """
 879     Callback function called when cherrypy.engine stops
 880     TODO: Ending database connections.
 881     """
 882     global vim_admin_thread
 883
 884     # terminate vim_admin_thread
 885     if vim_admin_thread:
 886         vim_admin_thread.terminate()
 887
 888     vim_admin_thread = None
 889     cherrypy.tree.apps["/ro"].root.ns.stop()
 890     cherrypy.log.error("Stopping osm_ng_ro")
 891
 892
 893 def ro_main(config_file):
 894     global ro_server
 895
 896     ro_server = Server()
 897     cherrypy.engine.subscribe("start", _start_service)
 898     cherrypy.engine.subscribe("stop", _stop_service)
 899     cherrypy.quickstart(ro_server, "/ro", config_file)
 900
 901
 902 def usage():
 903     print(
 904         """Usage: {} [options]
 905         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 906         -h|--help: shows this help
 907         """.format(
 908             sys.argv[0]
 909         )
 910     )
 911     # --log-socket-host HOST: send logs to this host")
 912     # --log-socket-port PORT: send logs using this port (default: 9022)")
 913
 914
 915 if __name__ == "__main__":
 916     try:
 917         # load parameters and configuration
 918         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 919         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 920         config_file = None
 921
 922         for o, a in opts:
 923             if o in ("-h", "--help"):
 924                 usage()
 925                 sys.exit()
 926             elif o in ("-c", "--config"):
 927                 config_file = a
 928             else:
 929                 raise ValueError("Unhandled option")
 930
 931         if config_file:
 932             if not path.isfile(config_file):
 933                 print(
 934                     "configuration file '{}' that not exist".format(config_file),
 935                     file=sys.stderr,
 936                 )
 937                 exit(1)
 938         else:
 939             for config_file in (
 940                 path.dirname(__file__) + "/ro.cfg",
 941                 "./ro.cfg",
 942                 "/etc/osm/ro.cfg",
 943             ):
 944                 if path.isfile(config_file):
 945                     break
 946             else:
 947                 print(
 948                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 949                     file=sys.stderr,
 950                 )
 951                 exit(1)
 952
 953         ro_main(config_file)
 954     except KeyboardInterrupt:
 955         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 956     except getopt.GetoptError as e:
 957         print(str(e), file=sys.stderr)
 958         # usage()
 959         exit(1)