NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "rebuild": {
  87                 "METHODS": ("POST",),
  88                 "ROLE_PERMISSION": "rebuild:",
  89                 "<ID>": {
  90                     "METHODS": ("POST",),
  91                     "ROLE_PERMISSION": "rebuild:id:",
  92                 },
  93             },
  94             "start": {
  95                 "METHODS": ("POST",),
  96                 "ROLE_PERMISSION": "start:",
  97                 "<ID>": {
  98                     "METHODS": ("POST",),
  99                     "ROLE_PERMISSION": "start:id:",
 100                 },
 101             },
 102             "stop": {
 103                 "METHODS": ("POST",),
 104                 "ROLE_PERMISSION": "stop:",
 105                 "<ID>": {
 106                     "METHODS": ("POST",),
 107                     "ROLE_PERMISSION": "stop:id:",
 108                 },
 109             },
 110             "deploy": {
 111                 "METHODS": ("GET",),
 112                 "ROLE_PERMISSION": "deploy:",
 113                 "<ID>": {
 114                     "METHODS": ("GET", "POST", "DELETE"),
 115                     "ROLE_PERMISSION": "deploy:id:",
 116                     "<ID>": {
 117                         "METHODS": ("GET",),
 118                         "ROLE_PERMISSION": "deploy:id:id:",
 119                         "cancel": {
 120                             "METHODS": ("POST",),
 121                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 122                         },
 123                     },
 124                 },
 125             },
 126             "recreate": {
 127                 "<ID>": {
 128                     "METHODS": ("POST"),
 129                     "ROLE_PERMISSION": "recreate:id:",
 130                     "<ID>": {
 131                         "METHODS": ("GET",),
 132                         "ROLE_PERMISSION": "recreate:id:id:",
 133                     },
 134                 },
 135             },
 136             "migrate": {
 137                 "<ID>": {
 138                     "METHODS": ("POST"),
 139                     "ROLE_PERMISSION": "migrate:id:",
 140                     "<ID>": {
 141                         "METHODS": ("GET",),
 142                         "ROLE_PERMISSION": "migrate:id:id:",
 143                     },
 144                 },
 145             },
 146             "verticalscale": {
 147                 "<ID>": {
 148                     "METHODS": ("POST"),
 149                     "ROLE_PERMISSION": "verticalscale:id:",
 150                     "<ID>": {
 151                         "METHODS": ("GET",),
 152                         "ROLE_PERMISSION": "verticalscale:id:id:",
 153                     },
 154                 },
 155             },
 156         }
 157     },
 158 }
 159
 160
 161 class RoException(Exception):
 162     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 163         Exception.__init__(self, message)
 164         self.http_code = http_code
 165
 166
 167 class AuthException(RoException):
 168     pass
 169
 170
 171 class Authenticator:
 172     def __init__(self, valid_url_methods, valid_query_string):
 173         self.valid_url_methods = valid_url_methods
 174         self.valid_query_string = valid_query_string
 175
 176     def authorize(self, *args, **kwargs):
 177         return {"token": "ok", "id": "ok"}
 178
 179     def new_token(self, token_info, indata, remote):
 180         return {"token": "ok", "id": "ok", "remote": remote}
 181
 182     def del_token(self, token_id):
 183         pass
 184
 185     def start(self, engine_config):
 186         pass
 187
 188
 189 class Server(object):
 190     instance = 0
 191     # to decode bytes to str
 192     reader = getreader("utf-8")
 193
 194     def __init__(self):
 195         self.instance += 1
 196         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 197         self.ns = Ns()
 198         self.map_operation = {
 199             "token:post": self.new_token,
 200             "token:id:delete": self.del_token,
 201             "deploy:get": self.ns.get_deploy,
 202             "deploy:id:get": self.ns.get_actions,
 203             "deploy:id:post": self.ns.deploy,
 204             "deploy:id:delete": self.ns.delete,
 205             "deploy:id:id:get": self.ns.status,
 206             "deploy:id:id:cancel:post": self.ns.cancel,
 207             "rebuild:id:post": self.ns.rebuild_start_stop,
 208             "start:id:post": self.ns.rebuild_start_stop,
 209             "stop:id:post": self.ns.rebuild_start_stop,
 210             "recreate:id:post": self.ns.recreate,
 211             "recreate:id:id:get": self.ns.recreate_status,
 212             "migrate:id:post": self.ns.migrate,
 213             "verticalscale:id:post": self.ns.verticalscale,
 214         }
 215
 216     def _format_in(self, kwargs):
 217         error_text = ""
 218         try:
 219             indata = None
 220
 221             if cherrypy.request.body.length:
 222                 error_text = "Invalid input format "
 223
 224                 if "Content-Type" in cherrypy.request.headers:
 225                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 226                         error_text = "Invalid json format "
 227                         indata = json.load(self.reader(cherrypy.request.body))
 228                         cherrypy.request.headers.pop("Content-File-MD5", None)
 229                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 230                         error_text = "Invalid yaml format "
 231                         indata = yaml.safe_load(cherrypy.request.body)
 232                         cherrypy.request.headers.pop("Content-File-MD5", None)
 233                     elif (
 234                         "application/binary" in cherrypy.request.headers["Content-Type"]
 235                         or "application/gzip"
 236                         in cherrypy.request.headers["Content-Type"]
 237                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 238                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 239                     ):
 240                         indata = cherrypy.request.body  # .read()
 241                     elif (
 242                         "multipart/form-data"
 243                         in cherrypy.request.headers["Content-Type"]
 244                     ):
 245                         if "descriptor_file" in kwargs:
 246                             filecontent = kwargs.pop("descriptor_file")
 247
 248                             if not filecontent.file:
 249                                 raise RoException(
 250                                     "empty file or content", HTTPStatus.BAD_REQUEST
 251                                 )
 252
 253                             indata = filecontent.file  # .read()
 254
 255                             if filecontent.content_type.value:
 256                                 cherrypy.request.headers[
 257                                     "Content-Type"
 258                                 ] = filecontent.content_type.value
 259                     else:
 260                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 261                         #                          "Only 'Content-Type' of type 'application/json' or
 262                         # 'application/yaml' for input format are available")
 263                         error_text = "Invalid yaml format "
 264                         indata = yaml.safe_load(cherrypy.request.body)
 265                         cherrypy.request.headers.pop("Content-File-MD5", None)
 266                 else:
 267                     error_text = "Invalid yaml format "
 268                     indata = yaml.safe_load(cherrypy.request.body)
 269                     cherrypy.request.headers.pop("Content-File-MD5", None)
 270
 271             if not indata:
 272                 indata = {}
 273
 274             format_yaml = False
 275             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 276                 format_yaml = True
 277
 278             for k, v in kwargs.items():
 279                 if isinstance(v, str):
 280                     if v == "":
 281                         kwargs[k] = None
 282                     elif format_yaml:
 283                         try:
 284                             kwargs[k] = yaml.safe_load(v)
 285                         except Exception as yaml_error:
 286                             logging.exception(
 287                                 f"{yaml_error} occured while parsing the yaml"
 288                             )
 289                     elif (
 290                         k.endswith(".gt")
 291                         or k.endswith(".lt")
 292                         or k.endswith(".gte")
 293                         or k.endswith(".lte")
 294                     ):
 295                         try:
 296                             kwargs[k] = int(v)
 297                         except Exception:
 298                             try:
 299                                 kwargs[k] = float(v)
 300                             except Exception as keyword_error:
 301                                 logging.exception(
 302                                     f"{keyword_error} occured while getting the keyword arguments"
 303                                 )
 304                     elif v.find(",") > 0:
 305                         kwargs[k] = v.split(",")
 306                 elif isinstance(v, (list, tuple)):
 307                     for index in range(0, len(v)):
 308                         if v[index] == "":
 309                             v[index] = None
 310                         elif format_yaml:
 311                             try:
 312                                 v[index] = yaml.safe_load(v[index])
 313                             except Exception as error:
 314                                 logging.exception(
 315                                     f"{error} occured while parsing the yaml"
 316                                 )
 317
 318             return indata
 319         except (ValueError, yaml.YAMLError) as exc:
 320             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 321         except KeyError as exc:
 322             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 323         except Exception as exc:
 324             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 325
 326     @staticmethod
 327     def _format_out(data, token_info=None, _format=None):
 328         """
 329         return string of dictionary data according to requested json, yaml, xml. By default json
 330         :param data: response to be sent. Can be a dict, text or file
 331         :param token_info: Contains among other username and project
 332         :param _format: The format to be set as Content-Type if data is a file
 333         :return: None
 334         """
 335         accept = cherrypy.request.headers.get("Accept")
 336
 337         if data is None:
 338             if accept and "text/html" in accept:
 339                 return html.format(
 340                     data, cherrypy.request, cherrypy.response, token_info
 341                 )
 342
 343             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 344             return
 345         elif hasattr(data, "read"):  # file object
 346             if _format:
 347                 cherrypy.response.headers["Content-Type"] = _format
 348             elif "b" in data.mode:  # binariy asssumig zip
 349                 cherrypy.response.headers["Content-Type"] = "application/zip"
 350             else:
 351                 cherrypy.response.headers["Content-Type"] = "text/plain"
 352
 353             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 354             return data
 355
 356         if accept:
 357             if "application/json" in accept:
 358                 cherrypy.response.headers[
 359                     "Content-Type"
 360                 ] = "application/json; charset=utf-8"
 361                 a = json.dumps(data, indent=4) + "\n"
 362
 363                 return a.encode("utf8")
 364             elif "text/html" in accept:
 365                 return html.format(
 366                     data, cherrypy.request, cherrypy.response, token_info
 367                 )
 368             elif (
 369                 "application/yaml" in accept
 370                 or "*/*" in accept
 371                 or "text/plain" in accept
 372             ):
 373                 pass
 374             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 375             elif cherrypy.response.status >= 400:
 376                 raise cherrypy.HTTPError(
 377                     HTTPStatus.NOT_ACCEPTABLE.value,
 378                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 379                     "for output format are available",
 380                 )
 381
 382         cherrypy.response.headers["Content-Type"] = "application/yaml"
 383
 384         return yaml.safe_dump(
 385             data,
 386             explicit_start=True,
 387             indent=4,
 388             default_flow_style=False,
 389             tags=False,
 390             encoding="utf-8",
 391             allow_unicode=True,
 392         )  # , canonical=True, default_style='"'
 393
 394     @cherrypy.expose
 395     def index(self, *args, **kwargs):
 396         token_info = None
 397
 398         try:
 399             if cherrypy.request.method == "GET":
 400                 token_info = self.authenticator.authorize()
 401                 outdata = token_info  # Home page
 402             else:
 403                 raise cherrypy.HTTPError(
 404                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 405                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 406                 )
 407
 408             return self._format_out(outdata, token_info)
 409         except (NsException, AuthException) as e:
 410             # cherrypy.log("index Exception {}".format(e))
 411             cherrypy.response.status = e.http_code.value
 412
 413             return self._format_out("Welcome to OSM!", token_info)
 414
 415     @cherrypy.expose
 416     def version(self, *args, **kwargs):
 417         # TODO consider to remove and provide version using the static version file
 418         try:
 419             if cherrypy.request.method != "GET":
 420                 raise RoException(
 421                     "Only method GET is allowed",
 422                     HTTPStatus.METHOD_NOT_ALLOWED,
 423                 )
 424             elif args or kwargs:
 425                 raise RoException(
 426                     "Invalid URL or query string for version",
 427                     HTTPStatus.METHOD_NOT_ALLOWED,
 428                 )
 429
 430             # TODO include version of other modules, pick up from some kafka admin message
 431             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 432
 433             return self._format_out(osm_ng_ro_version)
 434         except RoException as e:
 435             cherrypy.response.status = e.http_code.value
 436             problem_details = {
 437                 "code": e.http_code.name,
 438                 "status": e.http_code.value,
 439                 "detail": str(e),
 440             }
 441
 442             return self._format_out(problem_details, None)
 443
 444     def new_token(self, engine_session, indata, *args, **kwargs):
 445         token_info = None
 446
 447         try:
 448             token_info = self.authenticator.authorize()
 449         except Exception:
 450             token_info = None
 451
 452         if kwargs:
 453             indata.update(kwargs)
 454
 455         # This is needed to log the user when authentication fails
 456         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 457         token_info = self.authenticator.new_token(
 458             token_info, indata, cherrypy.request.remote
 459         )
 460         cherrypy.session["Authorization"] = token_info["id"]
 461         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 462         # for logging
 463
 464         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 465         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 466
 467         return token_info, token_info["id"], True
 468
 469     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 470         token_id = _id
 471
 472         if not token_id and "id" in kwargs:
 473             token_id = kwargs["id"]
 474         elif not token_id:
 475             token_info = self.authenticator.authorize()
 476             # for logging
 477             token_id = token_info["id"]
 478
 479         self.authenticator.del_token(token_id)
 480         token_info = None
 481         cherrypy.session["Authorization"] = "logout"
 482         # cherrypy.response.cookie["Authorization"] = token_id
 483         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 484
 485         return None, None, True
 486
 487     @cherrypy.expose
 488     def test(self, *args, **kwargs):
 489         if not cherrypy.config.get("server.enable_test") or (
 490             isinstance(cherrypy.config["server.enable_test"], str)
 491             and cherrypy.config["server.enable_test"].lower() == "false"
 492         ):
 493             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 494
 495             return "test URL is disabled"
 496
 497         thread_info = None
 498
 499         if args and args[0] == "help":
 500             return (
 501                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 502                 "sleep/<time>\nmessage/topic\n</pre></html>"
 503             )
 504         elif args and args[0] == "init":
 505             try:
 506                 # self.ns.load_dbase(cherrypy.request.app.config)
 507                 self.ns.create_admin()
 508
 509                 return "Done. User 'admin', password 'admin' created"
 510             except Exception:
 511                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 512
 513                 return self._format_out("Database already initialized")
 514         elif args and args[0] == "file":
 515             return cherrypy.lib.static.serve_file(
 516                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 517                 "text/plain",
 518                 "attachment",
 519             )
 520         elif args and args[0] == "file2":
 521             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 522             f = open(f_path, "r")
 523             cherrypy.response.headers["Content-type"] = "text/plain"
 524             return f
 525
 526         elif len(args) == 2 and args[0] == "db-clear":
 527             deleted_info = self.ns.db.del_list(args[1], kwargs)
 528             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 529         elif len(args) and args[0] == "fs-clear":
 530             if len(args) >= 2:
 531                 folders = (args[1],)
 532             else:
 533                 folders = self.ns.fs.dir_ls(".")
 534
 535             for folder in folders:
 536                 self.ns.fs.file_delete(folder)
 537
 538             return ",".join(folders) + " folders deleted\n"
 539         elif args and args[0] == "login":
 540             if not cherrypy.request.headers.get("Authorization"):
 541                 cherrypy.response.headers[
 542                     "WWW-Authenticate"
 543                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 544                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 545         elif args and args[0] == "login2":
 546             if not cherrypy.request.headers.get("Authorization"):
 547                 cherrypy.response.headers[
 548                     "WWW-Authenticate"
 549                 ] = 'Bearer realm="Access to OSM site"'
 550                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 551         elif args and args[0] == "sleep":
 552             sleep_time = 5
 553
 554             try:
 555                 sleep_time = int(args[1])
 556             except Exception:
 557                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 558                 return self._format_out("Database already initialized")
 559
 560             thread_info = cherrypy.thread_data
 561             print(thread_info)
 562             time.sleep(sleep_time)
 563             # thread_info
 564         elif len(args) >= 2 and args[0] == "message":
 565             main_topic = args[1]
 566             return_text = "<html><pre>{} ->\n".format(main_topic)
 567
 568             try:
 569                 if cherrypy.request.method == "POST":
 570                     to_send = yaml.safe_load(cherrypy.request.body)
 571                     for k, v in to_send.items():
 572                         self.ns.msg.write(main_topic, k, v)
 573                         return_text += "  {}: {}\n".format(k, v)
 574                 elif cherrypy.request.method == "GET":
 575                     for k, v in kwargs.items():
 576                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 577                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 578             except Exception as e:
 579                 return_text += "Error: " + str(e)
 580
 581             return_text += "</pre></html>\n"
 582
 583             return return_text
 584
 585         return_text = (
 586             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 587             + "  kwargs: {}\n".format(kwargs)
 588             + "  headers: {}\n".format(cherrypy.request.headers)
 589             + "  path_info: {}\n".format(cherrypy.request.path_info)
 590             + "  query_string: {}\n".format(cherrypy.request.query_string)
 591             + "  session: {}\n".format(cherrypy.session)
 592             + "  cookie: {}\n".format(cherrypy.request.cookie)
 593             + "  method: {}\n".format(cherrypy.request.method)
 594             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 595             + "  body:\n"
 596         )
 597         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 598
 599         if cherrypy.request.body.length:
 600             return_text += "    content: {}\n".format(
 601                 str(
 602                     cherrypy.request.body.read(
 603                         int(cherrypy.request.headers.get("Content-Length", 0))
 604                     )
 605                 )
 606             )
 607
 608         if thread_info:
 609             return_text += "thread: {}\n".format(thread_info)
 610
 611         return_text += "</pre></html>"
 612
 613         return return_text
 614
 615     @staticmethod
 616     def _check_valid_url_method(method, *args):
 617         if len(args) < 3:
 618             raise RoException(
 619                 "URL must contain at least 'main_topic/version/topic'",
 620                 HTTPStatus.METHOD_NOT_ALLOWED,
 621             )
 622
 623         reference = valid_url_methods
 624         for arg in args:
 625             if arg is None:
 626                 break
 627
 628             if not isinstance(reference, dict):
 629                 raise RoException(
 630                     "URL contains unexpected extra items '{}'".format(arg),
 631                     HTTPStatus.METHOD_NOT_ALLOWED,
 632                 )
 633
 634             if arg in reference:
 635                 reference = reference[arg]
 636             elif "<ID>" in reference:
 637                 reference = reference["<ID>"]
 638             elif "*" in reference:
 639                 # reference = reference["*"]
 640                 break
 641             else:
 642                 raise RoException(
 643                     "Unexpected URL item {}".format(arg),
 644                     HTTPStatus.METHOD_NOT_ALLOWED,
 645                 )
 646
 647         if "TODO" in reference and method in reference["TODO"]:
 648             raise RoException(
 649                 "Method {} not supported yet for this URL".format(method),
 650                 HTTPStatus.NOT_IMPLEMENTED,
 651             )
 652         elif "METHODS" not in reference or method not in reference["METHODS"]:
 653             raise RoException(
 654                 "Method {} not supported for this URL".format(method),
 655                 HTTPStatus.METHOD_NOT_ALLOWED,
 656             )
 657
 658         return reference["ROLE_PERMISSION"] + method.lower()
 659
 660     @staticmethod
 661     def _set_location_header(main_topic, version, topic, id):
 662         """
 663         Insert response header Location with the URL of created item base on URL params
 664         :param main_topic:
 665         :param version:
 666         :param topic:
 667         :param id:
 668         :return: None
 669         """
 670         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 671         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 672             main_topic, version, topic, id
 673         )
 674
 675         return
 676
 677     @cherrypy.expose
 678     def default(
 679         self,
 680         main_topic=None,
 681         version=None,
 682         topic=None,
 683         _id=None,
 684         _id2=None,
 685         *args,
 686         **kwargs,
 687     ):
 688         token_info = None
 689         outdata = None
 690         _format = None
 691         method = "DONE"
 692         rollback = []
 693         engine_session = None
 694
 695         try:
 696             if not main_topic or not version or not topic:
 697                 raise RoException(
 698                     "URL must contain at least 'main_topic/version/topic'",
 699                     HTTPStatus.METHOD_NOT_ALLOWED,
 700                 )
 701
 702             if main_topic not in (
 703                 "admin",
 704                 "ns",
 705             ):
 706                 raise RoException(
 707                     "URL main_topic '{}' not supported".format(main_topic),
 708                     HTTPStatus.METHOD_NOT_ALLOWED,
 709                 )
 710
 711             if version != "v1":
 712                 raise RoException(
 713                     "URL version '{}' not supported".format(version),
 714                     HTTPStatus.METHOD_NOT_ALLOWED,
 715                 )
 716
 717             if (
 718                 kwargs
 719                 and "METHOD" in kwargs
 720                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 721             ):
 722                 method = kwargs.pop("METHOD")
 723             else:
 724                 method = cherrypy.request.method
 725
 726             role_permission = self._check_valid_url_method(
 727                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 728             )
 729             # skip token validation if requesting a token
 730             indata = self._format_in(kwargs)
 731
 732             if main_topic != "admin" or topic != "tokens":
 733                 token_info = self.authenticator.authorize(role_permission, _id)
 734
 735             outdata, created_id, done = self.map_operation[role_permission](
 736                 engine_session, indata, version, _id, _id2, *args, *kwargs
 737             )
 738
 739             if created_id:
 740                 self._set_location_header(main_topic, version, topic, _id)
 741
 742             cherrypy.response.status = (
 743                 HTTPStatus.ACCEPTED.value
 744                 if not done
 745                 else HTTPStatus.OK.value
 746                 if outdata is not None
 747                 else HTTPStatus.NO_CONTENT.value
 748             )
 749
 750             return self._format_out(outdata, token_info, _format)
 751         except Exception as e:
 752             if isinstance(
 753                 e,
 754                 (
 755                     RoException,
 756                     NsException,
 757                     DbException,
 758                     FsException,
 759                     MsgException,
 760                     AuthException,
 761                     ValidationError,
 762                 ),
 763             ):
 764                 http_code_value = cherrypy.response.status = e.http_code.value
 765                 http_code_name = e.http_code.name
 766                 cherrypy.log("Exception {}".format(e))
 767             else:
 768                 http_code_value = (
 769                     cherrypy.response.status
 770                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 771                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 772                 http_code_name = HTTPStatus.BAD_REQUEST.name
 773
 774             if hasattr(outdata, "close"):  # is an open file
 775                 outdata.close()
 776
 777             error_text = str(e)
 778             rollback.reverse()
 779
 780             for rollback_item in rollback:
 781                 try:
 782                     if rollback_item.get("operation") == "set":
 783                         self.ns.db.set_one(
 784                             rollback_item["topic"],
 785                             {"_id": rollback_item["_id"]},
 786                             rollback_item["content"],
 787                             fail_on_empty=False,
 788                         )
 789                     else:
 790                         self.ns.db.del_one(
 791                             rollback_item["topic"],
 792                             {"_id": rollback_item["_id"]},
 793                             fail_on_empty=False,
 794                         )
 795                 except Exception as e2:
 796                     rollback_error_text = "Rollback Exception {}: {}".format(
 797                         rollback_item, e2
 798                     )
 799                     cherrypy.log(rollback_error_text)
 800                     error_text += ". " + rollback_error_text
 801
 802             # if isinstance(e, MsgException):
 803             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 804             #         engine_topic[:-1], method, error_text)
 805             problem_details = {
 806                 "code": http_code_name,
 807                 "status": http_code_value,
 808                 "detail": error_text,
 809             }
 810
 811             return self._format_out(problem_details, token_info)
 812             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 813         finally:
 814             if token_info:
 815                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 816                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 817                         if outdata.get(logging_id):
 818                             cherrypy.request.login += ";{}={}".format(
 819                                 logging_id, outdata[logging_id][:36]
 820                             )
 821
 822
 823 def _start_service():
 824     """
 825     Callback function called when cherrypy.engine starts
 826     Override configuration with env variables
 827     Set database, storage, message configuration
 828     Init database with admin/admin user password
 829     """
 830     global ro_server, vim_admin_thread
 831     # global vim_threads
 832     cherrypy.log.error("Starting osm_ng_ro")
 833     # update general cherrypy configuration
 834     update_dict = {}
 835     engine_config = cherrypy.tree.apps["/ro"].config
 836
 837     for k, v in environ.items():
 838         if not k.startswith("OSMRO_"):
 839             continue
 840
 841         k1, _, k2 = k[6:].lower().partition("_")
 842
 843         if not k2:
 844             continue
 845
 846         try:
 847             if k1 in ("server", "test", "auth", "log"):
 848                 # update [global] configuration
 849                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 850             elif k1 == "static":
 851                 # update [/static] configuration
 852                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 853             elif k1 == "tools":
 854                 # update [/] configuration
 855                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 856             elif k1 in ("message", "database", "storage", "authentication", "period"):
 857                 engine_config[k1][k2] = yaml.safe_load(v)
 858
 859         except Exception as e:
 860             raise RoException("Cannot load env '{}': {}".format(k, e))
 861
 862     if update_dict:
 863         cherrypy.config.update(update_dict)
 864         engine_config["global"].update(update_dict)
 865
 866     # logging cherrypy
 867     log_format_simple = (
 868         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 869     )
 870     log_formatter_simple = logging.Formatter(
 871         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 872     )
 873     logger_server = logging.getLogger("cherrypy.error")
 874     logger_access = logging.getLogger("cherrypy.access")
 875     logger_cherry = logging.getLogger("cherrypy")
 876     logger = logging.getLogger("ro")
 877
 878     if "log.file" in engine_config["global"]:
 879         file_handler = logging.handlers.RotatingFileHandler(
 880             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 881         )
 882         file_handler.setFormatter(log_formatter_simple)
 883         logger_cherry.addHandler(file_handler)
 884         logger.addHandler(file_handler)
 885
 886     # log always to standard output
 887     for format_, logger in {
 888         "ro.server %(filename)s:%(lineno)s": logger_server,
 889         "ro.access %(filename)s:%(lineno)s": logger_access,
 890         "%(name)s %(filename)s:%(lineno)s": logger,
 891     }.items():
 892         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 893         log_formatter_cherry = logging.Formatter(
 894             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 895         )
 896         str_handler = logging.StreamHandler()
 897         str_handler.setFormatter(log_formatter_cherry)
 898         logger.addHandler(str_handler)
 899
 900     if engine_config["global"].get("log.level"):
 901         logger_cherry.setLevel(engine_config["global"]["log.level"])
 902         logger.setLevel(engine_config["global"]["log.level"])
 903
 904     # logging other modules
 905     for k1, logname in {
 906         "message": "ro.msg",
 907         "database": "ro.db",
 908         "storage": "ro.fs",
 909     }.items():
 910         engine_config[k1]["logger_name"] = logname
 911         logger_module = logging.getLogger(logname)
 912
 913         if "logfile" in engine_config[k1]:
 914             file_handler = logging.handlers.RotatingFileHandler(
 915                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 916             )
 917             file_handler.setFormatter(log_formatter_simple)
 918             logger_module.addHandler(file_handler)
 919
 920         if "loglevel" in engine_config[k1]:
 921             logger_module.setLevel(engine_config[k1]["loglevel"])
 922     # TODO add more entries, e.g.: storage
 923
 924     engine_config["assignment"] = {}
 925     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 926     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 927     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 928     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 929
 930     # # start subscriptions thread:
 931     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 932     vim_admin_thread.start()
 933     # # Do not capture except SubscriptionException
 934
 935     # backend = engine_config["authentication"]["backend"]
 936     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 937     #                    .format(ro_version, ro_version_date, backend))
 938
 939
 940 def _stop_service():
 941     """
 942     Callback function called when cherrypy.engine stops
 943     TODO: Ending database connections.
 944     """
 945     global vim_admin_thread
 946
 947     # terminate vim_admin_thread
 948     if vim_admin_thread:
 949         vim_admin_thread.terminate()
 950
 951     vim_admin_thread = None
 952     cherrypy.tree.apps["/ro"].root.ns.stop()
 953     cherrypy.log.error("Stopping osm_ng_ro")
 954
 955
 956 def ro_main(config_file):
 957     global ro_server
 958
 959     ro_server = Server()
 960     cherrypy.engine.subscribe("start", _start_service)
 961     cherrypy.engine.subscribe("stop", _stop_service)
 962     cherrypy.quickstart(ro_server, "/ro", config_file)
 963
 964
 965 def usage():
 966     print(
 967         """Usage: {} [options]
 968         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 969         -h|--help: shows this help
 970         """.format(
 971             sys.argv[0]
 972         )
 973     )
 974     # --log-socket-host HOST: send logs to this host")
 975     # --log-socket-port PORT: send logs using this port (default: 9022)")
 976
 977
 978 if __name__ == "__main__":
 979     try:
 980         # load parameters and configuration
 981         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 982         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 983         config_file = None
 984
 985         for o, a in opts:
 986             if o in ("-h", "--help"):
 987                 usage()
 988                 sys.exit()
 989             elif o in ("-c", "--config"):
 990                 config_file = a
 991             else:
 992                 raise ValueError("Unhandled option")
 993
 994         if config_file:
 995             if not path.isfile(config_file):
 996                 print(
 997                     "configuration file '{}' that not exist".format(config_file),
 998                     file=sys.stderr,
 999                 )
1000                 exit(1)
1001         else:
1002             for config_file in (
1003                 path.dirname(__file__) + "/ro.cfg",
1004                 "./ro.cfg",
1005                 "/etc/osm/ro.cfg",
1006             ):
1007                 if path.isfile(config_file):
1008                     break
1009             else:
1010                 print(
1011                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1012                     file=sys.stderr,
1013                 )
1014                 exit(1)
1015
1016         ro_main(config_file)
1017     except KeyboardInterrupt:
1018         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1019     except getopt.GetoptError as e:
1020         print(str(e), file=sys.stderr)
1021         # usage()
1022         exit(1)