NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.monitor import start_monitoring, stop_monitoring
  39 from osm_ng_ro.ns import Ns, NsException
  40 from osm_ng_ro.validation import ValidationError
  41 from osm_ng_ro.vim_admin import VimAdminThread
  42 import yaml
  43
  44
  45 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  46 __version__ = "0.1."  # file version, not NBI version
  47 version_date = "May 2020"
  48
  49 database_version = "1.2"
  50 auth_database_version = "1.0"
  51 ro_server = None  # instance of Server class
  52 vim_admin_thread = None  # instance of VimAdminThread class
  53
  54 # vim_threads = None  # instance of VimThread class
  55
  56 """
  57 RO North Bound Interface
  58 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  59         /ns/v1/deploy                                           O
  60             /<nsrs_id>                                          O       O               O
  61                 /<action_id>                                    O
  62                     /cancel                                             O
  63
  64 """
  65
  66 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  67 # ^ Contains possible administrative query string words:
  68 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  69 #           (not owned by my session project).
  70 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  71 #     FORCE=True(by default)|False: Force edition/deletion operations
  72 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  73
  74 valid_url_methods = {
  75     # contains allowed URL and methods, and the role_permission name
  76     "admin": {
  77         "v1": {
  78             "tokens": {
  79                 "METHODS": ("POST",),
  80                 "ROLE_PERMISSION": "tokens:",
  81                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  82             },
  83         }
  84     },
  85     "ns": {
  86         "v1": {
  87             "rebuild": {
  88                 "METHODS": ("POST",),
  89                 "ROLE_PERMISSION": "rebuild:",
  90                 "<ID>": {
  91                     "METHODS": ("POST",),
  92                     "ROLE_PERMISSION": "rebuild:id:",
  93                 },
  94             },
  95             "start": {
  96                 "METHODS": ("POST",),
  97                 "ROLE_PERMISSION": "start:",
  98                 "<ID>": {
  99                     "METHODS": ("POST",),
 100                     "ROLE_PERMISSION": "start:id:",
 101                 },
 102             },
 103             "stop": {
 104                 "METHODS": ("POST",),
 105                 "ROLE_PERMISSION": "stop:",
 106                 "<ID>": {
 107                     "METHODS": ("POST",),
 108                     "ROLE_PERMISSION": "stop:id:",
 109                 },
 110             },
 111             "deploy": {
 112                 "METHODS": ("GET",),
 113                 "ROLE_PERMISSION": "deploy:",
 114                 "<ID>": {
 115                     "METHODS": ("GET", "POST", "DELETE"),
 116                     "ROLE_PERMISSION": "deploy:id:",
 117                     "<ID>": {
 118                         "METHODS": ("GET",),
 119                         "ROLE_PERMISSION": "deploy:id:id:",
 120                         "cancel": {
 121                             "METHODS": ("POST",),
 122                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 123                         },
 124                     },
 125                 },
 126             },
 127             "recreate": {
 128                 "<ID>": {
 129                     "METHODS": ("POST"),
 130                     "ROLE_PERMISSION": "recreate:id:",
 131                     "<ID>": {
 132                         "METHODS": ("GET",),
 133                         "ROLE_PERMISSION": "recreate:id:id:",
 134                     },
 135                 },
 136             },
 137             "migrate": {
 138                 "<ID>": {
 139                     "METHODS": ("POST"),
 140                     "ROLE_PERMISSION": "migrate:id:",
 141                     "<ID>": {
 142                         "METHODS": ("GET",),
 143                         "ROLE_PERMISSION": "migrate:id:id:",
 144                     },
 145                 },
 146             },
 147             "verticalscale": {
 148                 "<ID>": {
 149                     "METHODS": ("POST"),
 150                     "ROLE_PERMISSION": "verticalscale:id:",
 151                     "<ID>": {
 152                         "METHODS": ("GET",),
 153                         "ROLE_PERMISSION": "verticalscale:id:id:",
 154                     },
 155                 },
 156             },
 157         }
 158     },
 159 }
 160
 161
 162 class RoException(Exception):
 163     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 164         Exception.__init__(self, message)
 165         self.http_code = http_code
 166
 167
 168 class AuthException(RoException):
 169     pass
 170
 171
 172 class Authenticator:
 173     def __init__(self, valid_url_methods, valid_query_string):
 174         self.valid_url_methods = valid_url_methods
 175         self.valid_query_string = valid_query_string
 176
 177     def authorize(self, *args, **kwargs):
 178         return {"token": "ok", "id": "ok"}
 179
 180     def new_token(self, token_info, indata, remote):
 181         return {"token": "ok", "id": "ok", "remote": remote}
 182
 183     def del_token(self, token_id):
 184         pass
 185
 186     def start(self, engine_config):
 187         pass
 188
 189
 190 class Server(object):
 191     instance = 0
 192     # to decode bytes to str
 193     reader = getreader("utf-8")
 194
 195     def __init__(self):
 196         self.instance += 1
 197         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 198         self.ns = Ns()
 199         self.map_operation = {
 200             "token:post": self.new_token,
 201             "token:id:delete": self.del_token,
 202             "deploy:get": self.ns.get_deploy,
 203             "deploy:id:get": self.ns.get_actions,
 204             "deploy:id:post": self.ns.deploy,
 205             "deploy:id:delete": self.ns.delete,
 206             "deploy:id:id:get": self.ns.status,
 207             "deploy:id:id:cancel:post": self.ns.cancel,
 208             "rebuild:id:post": self.ns.rebuild_start_stop,
 209             "start:id:post": self.ns.rebuild_start_stop,
 210             "stop:id:post": self.ns.rebuild_start_stop,
 211             "recreate:id:post": self.ns.recreate,
 212             "recreate:id:id:get": self.ns.recreate_status,
 213             "migrate:id:post": self.ns.migrate,
 214             "verticalscale:id:post": self.ns.verticalscale,
 215         }
 216
 217     def _format_in(self, kwargs):
 218         error_text = ""
 219         try:
 220             indata = None
 221
 222             if cherrypy.request.body.length:
 223                 error_text = "Invalid input format "
 224
 225                 if "Content-Type" in cherrypy.request.headers:
 226                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 227                         error_text = "Invalid json format "
 228                         indata = json.load(self.reader(cherrypy.request.body))
 229                         cherrypy.request.headers.pop("Content-File-MD5", None)
 230                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 231                         error_text = "Invalid yaml format "
 232                         indata = yaml.safe_load(cherrypy.request.body)
 233                         cherrypy.request.headers.pop("Content-File-MD5", None)
 234                     elif (
 235                         "application/binary" in cherrypy.request.headers["Content-Type"]
 236                         or "application/gzip"
 237                         in cherrypy.request.headers["Content-Type"]
 238                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 239                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 240                     ):
 241                         indata = cherrypy.request.body  # .read()
 242                     elif (
 243                         "multipart/form-data"
 244                         in cherrypy.request.headers["Content-Type"]
 245                     ):
 246                         if "descriptor_file" in kwargs:
 247                             filecontent = kwargs.pop("descriptor_file")
 248
 249                             if not filecontent.file:
 250                                 raise RoException(
 251                                     "empty file or content", HTTPStatus.BAD_REQUEST
 252                                 )
 253
 254                             indata = filecontent.file  # .read()
 255
 256                             if filecontent.content_type.value:
 257                                 cherrypy.request.headers["Content-Type"] = (
 258                                     filecontent.content_type.value
 259                                 )
 260                     else:
 261                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 262                         #                          "Only 'Content-Type' of type 'application/json' or
 263                         # 'application/yaml' for input format are available")
 264                         error_text = "Invalid yaml format "
 265                         indata = yaml.safe_load(cherrypy.request.body)
 266                         cherrypy.request.headers.pop("Content-File-MD5", None)
 267                 else:
 268                     error_text = "Invalid yaml format "
 269                     indata = yaml.safe_load(cherrypy.request.body)
 270                     cherrypy.request.headers.pop("Content-File-MD5", None)
 271
 272             if not indata:
 273                 indata = {}
 274
 275             format_yaml = False
 276             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 277                 format_yaml = True
 278
 279             for k, v in kwargs.items():
 280                 if isinstance(v, str):
 281                     if v == "":
 282                         kwargs[k] = None
 283                     elif format_yaml:
 284                         try:
 285                             kwargs[k] = yaml.safe_load(v)
 286                         except Exception as yaml_error:
 287                             logging.exception(
 288                                 f"{yaml_error} occured while parsing the yaml"
 289                             )
 290                     elif (
 291                         k.endswith(".gt")
 292                         or k.endswith(".lt")
 293                         or k.endswith(".gte")
 294                         or k.endswith(".lte")
 295                     ):
 296                         try:
 297                             kwargs[k] = int(v)
 298                         except Exception:
 299                             try:
 300                                 kwargs[k] = float(v)
 301                             except Exception as keyword_error:
 302                                 logging.exception(
 303                                     f"{keyword_error} occured while getting the keyword arguments"
 304                                 )
 305                     elif v.find(",") > 0:
 306                         kwargs[k] = v.split(",")
 307                 elif isinstance(v, (list, tuple)):
 308                     for index in range(0, len(v)):
 309                         if v[index] == "":
 310                             v[index] = None
 311                         elif format_yaml:
 312                             try:
 313                                 v[index] = yaml.safe_load(v[index])
 314                             except Exception as error:
 315                                 logging.exception(
 316                                     f"{error} occured while parsing the yaml"
 317                                 )
 318
 319             return indata
 320         except (ValueError, yaml.YAMLError) as exc:
 321             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 322         except KeyError as exc:
 323             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 324         except Exception as exc:
 325             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 326
 327     @staticmethod
 328     def _format_out(data, token_info=None, _format=None):
 329         """
 330         return string of dictionary data according to requested json, yaml, xml. By default json
 331         :param data: response to be sent. Can be a dict, text or file
 332         :param token_info: Contains among other username and project
 333         :param _format: The format to be set as Content-Type if data is a file
 334         :return: None
 335         """
 336         accept = cherrypy.request.headers.get("Accept")
 337
 338         if data is None:
 339             if accept and "text/html" in accept:
 340                 return html.format(
 341                     data, cherrypy.request, cherrypy.response, token_info
 342                 )
 343
 344             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 345             return
 346         elif hasattr(data, "read"):  # file object
 347             if _format:
 348                 cherrypy.response.headers["Content-Type"] = _format
 349             elif "b" in data.mode:  # binariy asssumig zip
 350                 cherrypy.response.headers["Content-Type"] = "application/zip"
 351             else:
 352                 cherrypy.response.headers["Content-Type"] = "text/plain"
 353
 354             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 355             return data
 356
 357         if accept:
 358             if "application/json" in accept:
 359                 cherrypy.response.headers["Content-Type"] = (
 360                     "application/json; charset=utf-8"
 361                 )
 362                 a = json.dumps(data, indent=4) + "\n"
 363
 364                 return a.encode("utf8")
 365             elif "text/html" in accept:
 366                 return html.format(
 367                     data, cherrypy.request, cherrypy.response, token_info
 368                 )
 369             elif (
 370                 "application/yaml" in accept
 371                 or "*/*" in accept
 372                 or "text/plain" in accept
 373             ):
 374                 pass
 375             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 376             elif cherrypy.response.status >= 400:
 377                 raise cherrypy.HTTPError(
 378                     HTTPStatus.NOT_ACCEPTABLE.value,
 379                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 380                     "for output format are available",
 381                 )
 382
 383         cherrypy.response.headers["Content-Type"] = "application/yaml"
 384
 385         return yaml.safe_dump(
 386             data,
 387             explicit_start=True,
 388             indent=4,
 389             default_flow_style=False,
 390             tags=False,
 391             encoding="utf-8",
 392             allow_unicode=True,
 393         )  # , canonical=True, default_style='"'
 394
 395     @cherrypy.expose
 396     def index(self, *args, **kwargs):
 397         token_info = None
 398
 399         try:
 400             if cherrypy.request.method == "GET":
 401                 token_info = self.authenticator.authorize()
 402                 outdata = token_info  # Home page
 403             else:
 404                 raise cherrypy.HTTPError(
 405                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 406                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 407                 )
 408
 409             return self._format_out(outdata, token_info)
 410         except (NsException, AuthException) as e:
 411             # cherrypy.log("index Exception {}".format(e))
 412             cherrypy.response.status = e.http_code.value
 413
 414             return self._format_out("Welcome to OSM!", token_info)
 415
 416     @cherrypy.expose
 417     def version(self, *args, **kwargs):
 418         # TODO consider to remove and provide version using the static version file
 419         try:
 420             if cherrypy.request.method != "GET":
 421                 raise RoException(
 422                     "Only method GET is allowed",
 423                     HTTPStatus.METHOD_NOT_ALLOWED,
 424                 )
 425             elif args or kwargs:
 426                 raise RoException(
 427                     "Invalid URL or query string for version",
 428                     HTTPStatus.METHOD_NOT_ALLOWED,
 429                 )
 430
 431             # TODO include version of other modules, pick up from some kafka admin message
 432             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 433
 434             return self._format_out(osm_ng_ro_version)
 435         except RoException as e:
 436             cherrypy.response.status = e.http_code.value
 437             problem_details = {
 438                 "code": e.http_code.name,
 439                 "status": e.http_code.value,
 440                 "detail": str(e),
 441             }
 442
 443             return self._format_out(problem_details, None)
 444
 445     def new_token(self, engine_session, indata, *args, **kwargs):
 446         token_info = None
 447
 448         try:
 449             token_info = self.authenticator.authorize()
 450         except Exception:
 451             token_info = None
 452
 453         if kwargs:
 454             indata.update(kwargs)
 455
 456         # This is needed to log the user when authentication fails
 457         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 458         token_info = self.authenticator.new_token(
 459             token_info, indata, cherrypy.request.remote
 460         )
 461         cherrypy.session["Authorization"] = token_info["id"]
 462         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 463         # for logging
 464
 465         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 466         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 467
 468         return token_info, token_info["id"], True
 469
 470     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 471         token_id = _id
 472
 473         if not token_id and "id" in kwargs:
 474             token_id = kwargs["id"]
 475         elif not token_id:
 476             token_info = self.authenticator.authorize()
 477             # for logging
 478             token_id = token_info["id"]
 479
 480         self.authenticator.del_token(token_id)
 481         token_info = None
 482         cherrypy.session["Authorization"] = "logout"
 483         # cherrypy.response.cookie["Authorization"] = token_id
 484         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 485
 486         return None, None, True
 487
 488     @cherrypy.expose
 489     def test(self, *args, **kwargs):
 490         if not cherrypy.config.get("server.enable_test") or (
 491             isinstance(cherrypy.config["server.enable_test"], str)
 492             and cherrypy.config["server.enable_test"].lower() == "false"
 493         ):
 494             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 495
 496             return "test URL is disabled"
 497
 498         thread_info = None
 499
 500         if args and args[0] == "help":
 501             return (
 502                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 503                 "sleep/<time>\nmessage/topic\n</pre></html>"
 504             )
 505         elif args and args[0] == "init":
 506             try:
 507                 # self.ns.load_dbase(cherrypy.request.app.config)
 508                 self.ns.create_admin()
 509
 510                 return "Done. User 'admin', password 'admin' created"
 511             except Exception:
 512                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 513
 514                 return self._format_out("Database already initialized")
 515         elif args and args[0] == "file":
 516             return cherrypy.lib.static.serve_file(
 517                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 518                 "text/plain",
 519                 "attachment",
 520             )
 521         elif args and args[0] == "file2":
 522             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 523             f = open(f_path, "r")
 524             cherrypy.response.headers["Content-type"] = "text/plain"
 525             return f
 526
 527         elif len(args) == 2 and args[0] == "db-clear":
 528             deleted_info = self.ns.db.del_list(args[1], kwargs)
 529             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 530         elif len(args) and args[0] == "fs-clear":
 531             if len(args) >= 2:
 532                 folders = (args[1],)
 533             else:
 534                 folders = self.ns.fs.dir_ls(".")
 535
 536             for folder in folders:
 537                 self.ns.fs.file_delete(folder)
 538
 539             return ",".join(folders) + " folders deleted\n"
 540         elif args and args[0] == "login":
 541             if not cherrypy.request.headers.get("Authorization"):
 542                 cherrypy.response.headers["WWW-Authenticate"] = (
 543                     'Basic realm="Access to OSM site", charset="UTF-8"'
 544                 )
 545                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 546         elif args and args[0] == "login2":
 547             if not cherrypy.request.headers.get("Authorization"):
 548                 cherrypy.response.headers["WWW-Authenticate"] = (
 549                     'Bearer realm="Access to OSM site"'
 550                 )
 551                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 552         elif args and args[0] == "sleep":
 553             sleep_time = 5
 554
 555             try:
 556                 sleep_time = int(args[1])
 557             except Exception:
 558                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 559                 return self._format_out("Database already initialized")
 560
 561             thread_info = cherrypy.thread_data
 562             print(thread_info)
 563             time.sleep(sleep_time)
 564             # thread_info
 565         elif len(args) >= 2 and args[0] == "message":
 566             main_topic = args[1]
 567             return_text = "<html><pre>{} ->\n".format(main_topic)
 568
 569             try:
 570                 if cherrypy.request.method == "POST":
 571                     to_send = yaml.safe_load(cherrypy.request.body)
 572                     for k, v in to_send.items():
 573                         self.ns.msg.write(main_topic, k, v)
 574                         return_text += "  {}: {}\n".format(k, v)
 575                 elif cherrypy.request.method == "GET":
 576                     for k, v in kwargs.items():
 577                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 578                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 579             except Exception as e:
 580                 return_text += "Error: " + str(e)
 581
 582             return_text += "</pre></html>\n"
 583
 584             return return_text
 585
 586         return_text = (
 587             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 588             + "  kwargs: {}\n".format(kwargs)
 589             + "  headers: {}\n".format(cherrypy.request.headers)
 590             + "  path_info: {}\n".format(cherrypy.request.path_info)
 591             + "  query_string: {}\n".format(cherrypy.request.query_string)
 592             + "  session: {}\n".format(cherrypy.session)
 593             + "  cookie: {}\n".format(cherrypy.request.cookie)
 594             + "  method: {}\n".format(cherrypy.request.method)
 595             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 596             + "  body:\n"
 597         )
 598         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 599
 600         if cherrypy.request.body.length:
 601             return_text += "    content: {}\n".format(
 602                 str(
 603                     cherrypy.request.body.read(
 604                         int(cherrypy.request.headers.get("Content-Length", 0))
 605                     )
 606                 )
 607             )
 608
 609         if thread_info:
 610             return_text += "thread: {}\n".format(thread_info)
 611
 612         return_text += "</pre></html>"
 613
 614         return return_text
 615
 616     @staticmethod
 617     def _check_valid_url_method(method, *args):
 618         if len(args) < 3:
 619             raise RoException(
 620                 "URL must contain at least 'main_topic/version/topic'",
 621                 HTTPStatus.METHOD_NOT_ALLOWED,
 622             )
 623
 624         reference = valid_url_methods
 625         for arg in args:
 626             if arg is None:
 627                 break
 628
 629             if not isinstance(reference, dict):
 630                 raise RoException(
 631                     "URL contains unexpected extra items '{}'".format(arg),
 632                     HTTPStatus.METHOD_NOT_ALLOWED,
 633                 )
 634
 635             if arg in reference:
 636                 reference = reference[arg]
 637             elif "<ID>" in reference:
 638                 reference = reference["<ID>"]
 639             elif "*" in reference:
 640                 # reference = reference["*"]
 641                 break
 642             else:
 643                 raise RoException(
 644                     "Unexpected URL item {}".format(arg),
 645                     HTTPStatus.METHOD_NOT_ALLOWED,
 646                 )
 647
 648         if "TODO" in reference and method in reference["TODO"]:
 649             raise RoException(
 650                 "Method {} not supported yet for this URL".format(method),
 651                 HTTPStatus.NOT_IMPLEMENTED,
 652             )
 653         elif "METHODS" not in reference or method not in reference["METHODS"]:
 654             raise RoException(
 655                 "Method {} not supported for this URL".format(method),
 656                 HTTPStatus.METHOD_NOT_ALLOWED,
 657             )
 658
 659         return reference["ROLE_PERMISSION"] + method.lower()
 660
 661     @staticmethod
 662     def _set_location_header(main_topic, version, topic, id):
 663         """
 664         Insert response header Location with the URL of created item base on URL params
 665         :param main_topic:
 666         :param version:
 667         :param topic:
 668         :param id:
 669         :return: None
 670         """
 671         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 672         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 673             main_topic, version, topic, id
 674         )
 675
 676         return
 677
 678     @cherrypy.expose
 679     def default(
 680         self,
 681         main_topic=None,
 682         version=None,
 683         topic=None,
 684         _id=None,
 685         _id2=None,
 686         *args,
 687         **kwargs,
 688     ):
 689         token_info = None
 690         outdata = {}
 691         _format = None
 692         method = "DONE"
 693         rollback = []
 694         engine_session = None
 695
 696         try:
 697             if not main_topic or not version or not topic:
 698                 raise RoException(
 699                     "URL must contain at least 'main_topic/version/topic'",
 700                     HTTPStatus.METHOD_NOT_ALLOWED,
 701                 )
 702
 703             if main_topic not in (
 704                 "admin",
 705                 "ns",
 706             ):
 707                 raise RoException(
 708                     "URL main_topic '{}' not supported".format(main_topic),
 709                     HTTPStatus.METHOD_NOT_ALLOWED,
 710                 )
 711
 712             if version != "v1":
 713                 raise RoException(
 714                     "URL version '{}' not supported".format(version),
 715                     HTTPStatus.METHOD_NOT_ALLOWED,
 716                 )
 717
 718             if (
 719                 kwargs
 720                 and "METHOD" in kwargs
 721                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 722             ):
 723                 method = kwargs.pop("METHOD")
 724             else:
 725                 method = cherrypy.request.method
 726
 727             role_permission = self._check_valid_url_method(
 728                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 729             )
 730             # skip token validation if requesting a token
 731             indata = self._format_in(kwargs)
 732
 733             if main_topic != "admin" or topic != "tokens":
 734                 token_info = self.authenticator.authorize(role_permission, _id)
 735
 736             outdata, created_id, done = self.map_operation[role_permission](
 737                 engine_session, indata, version, _id, _id2, *args, *kwargs
 738             )
 739
 740             if created_id:
 741                 self._set_location_header(main_topic, version, topic, _id)
 742
 743             cherrypy.response.status = (
 744                 HTTPStatus.ACCEPTED.value
 745                 if not done
 746                 else (
 747                     HTTPStatus.OK.value
 748                     if outdata is not None
 749                     else HTTPStatus.NO_CONTENT.value
 750                 )
 751             )
 752
 753             return self._format_out(outdata, token_info, _format)
 754         except Exception as e:
 755             if isinstance(
 756                 e,
 757                 (
 758                     RoException,
 759                     NsException,
 760                     DbException,
 761                     FsException,
 762                     MsgException,
 763                     AuthException,
 764                     ValidationError,
 765                 ),
 766             ):
 767                 http_code_value = cherrypy.response.status = e.http_code.value
 768                 http_code_name = e.http_code.name
 769                 cherrypy.log("Exception {}".format(e))
 770             else:
 771                 http_code_value = cherrypy.response.status = (
 772                     HTTPStatus.BAD_REQUEST.value
 773                 )  # INTERNAL_SERVER_ERROR
 774                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 775                 http_code_name = HTTPStatus.BAD_REQUEST.name
 776
 777             if hasattr(outdata, "close"):  # is an open file
 778                 outdata.close()
 779
 780             error_text = str(e)
 781             rollback.reverse()
 782
 783             for rollback_item in rollback:
 784                 try:
 785                     if rollback_item.get("operation") == "set":
 786                         self.ns.db.set_one(
 787                             rollback_item["topic"],
 788                             {"_id": rollback_item["_id"]},
 789                             rollback_item["content"],
 790                             fail_on_empty=False,
 791                         )
 792                     else:
 793                         self.ns.db.del_one(
 794                             rollback_item["topic"],
 795                             {"_id": rollback_item["_id"]},
 796                             fail_on_empty=False,
 797                         )
 798                 except Exception as e2:
 799                     rollback_error_text = "Rollback Exception {}: {}".format(
 800                         rollback_item, e2
 801                     )
 802                     cherrypy.log(rollback_error_text)
 803                     error_text += ". " + rollback_error_text
 804
 805             # if isinstance(e, MsgException):
 806             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 807             #         engine_topic[:-1], method, error_text)
 808             problem_details = {
 809                 "code": http_code_name,
 810                 "status": http_code_value,
 811                 "detail": error_text,
 812             }
 813
 814             return self._format_out(problem_details, token_info)
 815             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 816         finally:
 817             if token_info:
 818                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 819                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 820                         if outdata.get(logging_id):
 821                             cherrypy.request.login += ";{}={}".format(
 822                                 logging_id, outdata[logging_id][:36]
 823                             )
 824
 825
 826 def _start_service():
 827     """
 828     Callback function called when cherrypy.engine starts
 829     Override configuration with env variables
 830     Set database, storage, message configuration
 831     Init database with admin/admin user password
 832     """
 833     global ro_server, vim_admin_thread
 834     # global vim_threads
 835     cherrypy.log.error("Starting osm_ng_ro")
 836     # update general cherrypy configuration
 837     update_dict = {}
 838     engine_config = cherrypy.tree.apps["/ro"].config
 839
 840     for k, v in environ.items():
 841         if not k.startswith("OSMRO_"):
 842             continue
 843
 844         k1, _, k2 = k[6:].lower().partition("_")
 845
 846         if not k2:
 847             continue
 848
 849         try:
 850             if k1 in ("server", "test", "auth", "log"):
 851                 # update [global] configuration
 852                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 853             elif k1 == "static":
 854                 # update [/static] configuration
 855                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 856             elif k1 == "tools":
 857                 # update [/] configuration
 858                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 859             elif k1 in ("message", "database", "storage", "authentication", "period"):
 860                 engine_config[k1][k2] = yaml.safe_load(v)
 861
 862         except Exception as e:
 863             raise RoException("Cannot load env '{}': {}".format(k, e))
 864
 865     if update_dict:
 866         cherrypy.config.update(update_dict)
 867         engine_config["global"].update(update_dict)
 868
 869     # logging cherrypy
 870     log_format_simple = (
 871         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 872     )
 873     log_formatter_simple = logging.Formatter(
 874         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 875     )
 876     logger_server = logging.getLogger("cherrypy.error")
 877     logger_access = logging.getLogger("cherrypy.access")
 878     logger_cherry = logging.getLogger("cherrypy")
 879     logger = logging.getLogger("ro")
 880
 881     if "log.file" in engine_config["global"]:
 882         file_handler = logging.handlers.RotatingFileHandler(
 883             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 884         )
 885         file_handler.setFormatter(log_formatter_simple)
 886         logger_cherry.addHandler(file_handler)
 887         logger.addHandler(file_handler)
 888
 889     # log always to standard output
 890     for format_, logger in {
 891         "ro.server %(filename)s:%(lineno)s": logger_server,
 892         "ro.access %(filename)s:%(lineno)s": logger_access,
 893         "%(name)s %(filename)s:%(lineno)s": logger,
 894     }.items():
 895         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 896         log_formatter_cherry = logging.Formatter(
 897             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 898         )
 899         str_handler = logging.StreamHandler()
 900         str_handler.setFormatter(log_formatter_cherry)
 901         logger.addHandler(str_handler)
 902
 903     if engine_config["global"].get("log.level"):
 904         logger_cherry.setLevel(engine_config["global"]["log.level"])
 905         logger.setLevel(engine_config["global"]["log.level"])
 906
 907     # logging other modules
 908     for k1, logname in {
 909         "message": "ro.msg",
 910         "database": "ro.db",
 911         "storage": "ro.fs",
 912     }.items():
 913         engine_config[k1]["logger_name"] = logname
 914         logger_module = logging.getLogger(logname)
 915
 916         if "logfile" in engine_config[k1]:
 917             file_handler = logging.handlers.RotatingFileHandler(
 918                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 919             )
 920             file_handler.setFormatter(log_formatter_simple)
 921             logger_module.addHandler(file_handler)
 922
 923         if "loglevel" in engine_config[k1]:
 924             logger_module.setLevel(engine_config[k1]["loglevel"])
 925     # TODO add more entries, e.g.: storage
 926
 927     engine_config["assignment"] = {}
 928     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 929     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 930     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 931     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 932
 933     # # start subscriptions thread:
 934     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 935     vim_admin_thread.start()
 936     start_monitoring(config=engine_config)
 937
 938     # # Do not capture except SubscriptionException
 939
 940     # backend = engine_config["authentication"]["backend"]
 941     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 942     #                    .format(ro_version, ro_version_date, backend))
 943
 944
 945 def _stop_service():
 946     """
 947     Callback function called when cherrypy.engine stops
 948     TODO: Ending database connections.
 949     """
 950     global vim_admin_thread
 951
 952     # terminate vim_admin_thread
 953     if vim_admin_thread:
 954         vim_admin_thread.terminate()
 955     stop_monitoring()
 956     vim_admin_thread = None
 957     cherrypy.tree.apps["/ro"].root.ns.stop()
 958     cherrypy.log.error("Stopping osm_ng_ro")
 959
 960
 961 def ro_main(config_file):
 962     global ro_server
 963
 964     ro_server = Server()
 965     cherrypy.engine.subscribe("start", _start_service)
 966     cherrypy.engine.subscribe("stop", _stop_service)
 967     cherrypy.quickstart(ro_server, "/ro", config_file)
 968
 969
 970 def usage():
 971     print(
 972         """Usage: {} [options]
 973         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 974         -h|--help: shows this help
 975         """.format(
 976             sys.argv[0]
 977         )
 978     )
 979     # --log-socket-host HOST: send logs to this host")
 980     # --log-socket-port PORT: send logs using this port (default: 9022)")
 981
 982
 983 if __name__ == "__main__":
 984     try:
 985         # load parameters and configuration
 986         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 987         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 988         config_file = None
 989
 990         for o, a in opts:
 991             if o in ("-h", "--help"):
 992                 usage()
 993                 sys.exit()
 994             elif o in ("-c", "--config"):
 995                 config_file = a
 996             else:
 997                 raise ValueError("Unhandled option")
 998
 999         if config_file:
1000             if not path.isfile(config_file):
1001                 print(
1002                     "configuration file '{}' that not exist".format(config_file),
1003                     file=sys.stderr,
1004                 )
1005                 exit(1)
1006         else:
1007             for config_file in (
1008                 path.dirname(__file__) + "/ro.cfg",
1009                 "./ro.cfg",
1010                 "/etc/osm/ro.cfg",
1011             ):
1012                 if path.isfile(config_file):
1013                     break
1014             else:
1015                 print(
1016                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1017                     file=sys.stderr,
1018                 )
1019                 exit(1)
1020
1021         ro_main(config_file)
1022     except KeyboardInterrupt:
1023         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1024     except getopt.GetoptError as e:
1025         print(str(e), file=sys.stderr)
1026         # usage()
1027         exit(1)