NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.monitor import start_monitoring, stop_monitoring
  39 from osm_ng_ro.ns import Ns, NsException
  40 from osm_ng_ro.validation import ValidationError
  41 from osm_ng_ro.vim_admin import VimAdminThread
  42 import yaml
  43
  44
  45 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  46 __version__ = "0.1."  # file version, not NBI version
  47 version_date = "May 2020"
  48
  49 database_version = "1.2"
  50 auth_database_version = "1.0"
  51 ro_server = None  # instance of Server class
  52 vim_admin_thread = None  # instance of VimAdminThread class
  53
  54 # vim_threads = None  # instance of VimThread class
  55
  56 """
  57 RO North Bound Interface
  58 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  59         /ns/v1/deploy                                           O
  60             /<nsrs_id>                                          O       O               O
  61                 /<action_id>                                    O
  62                     /cancel                                             O
  63
  64 """
  65
  66 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  67 # ^ Contains possible administrative query string words:
  68 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  69 #           (not owned by my session project).
  70 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  71 #     FORCE=True(by default)|False: Force edition/deletion operations
  72 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  73
  74 valid_url_methods = {
  75     # contains allowed URL and methods, and the role_permission name
  76     "admin": {
  77         "v1": {
  78             "tokens": {
  79                 "METHODS": ("POST",),
  80                 "ROLE_PERMISSION": "tokens:",
  81                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  82             },
  83         }
  84     },
  85     "ns": {
  86         "v1": {
  87             "rebuild": {
  88                 "METHODS": ("POST",),
  89                 "ROLE_PERMISSION": "rebuild:",
  90                 "<ID>": {
  91                     "METHODS": ("POST",),
  92                     "ROLE_PERMISSION": "rebuild:id:",
  93                 },
  94             },
  95             "start": {
  96                 "METHODS": ("POST",),
  97                 "ROLE_PERMISSION": "start:",
  98                 "<ID>": {
  99                     "METHODS": ("POST",),
 100                     "ROLE_PERMISSION": "start:id:",
 101                 },
 102             },
 103             "stop": {
 104                 "METHODS": ("POST",),
 105                 "ROLE_PERMISSION": "stop:",
 106                 "<ID>": {
 107                     "METHODS": ("POST",),
 108                     "ROLE_PERMISSION": "stop:id:",
 109                 },
 110             },
 111             "deploy": {
 112                 "METHODS": ("GET",),
 113                 "ROLE_PERMISSION": "deploy:",
 114                 "<ID>": {
 115                     "METHODS": ("GET", "POST", "DELETE"),
 116                     "ROLE_PERMISSION": "deploy:id:",
 117                     "<ID>": {
 118                         "METHODS": ("GET",),
 119                         "ROLE_PERMISSION": "deploy:id:id:",
 120                         "cancel": {
 121                             "METHODS": ("POST",),
 122                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 123                         },
 124                     },
 125                 },
 126             },
 127             "recreate": {
 128                 "<ID>": {
 129                     "METHODS": ("POST"),
 130                     "ROLE_PERMISSION": "recreate:id:",
 131                     "<ID>": {
 132                         "METHODS": ("GET",),
 133                         "ROLE_PERMISSION": "recreate:id:id:",
 134                     },
 135                 },
 136             },
 137             "migrate": {
 138                 "<ID>": {
 139                     "METHODS": ("POST"),
 140                     "ROLE_PERMISSION": "migrate:id:",
 141                     "<ID>": {
 142                         "METHODS": ("GET",),
 143                         "ROLE_PERMISSION": "migrate:id:id:",
 144                     },
 145                 },
 146             },
 147             "verticalscale": {
 148                 "<ID>": {
 149                     "METHODS": ("POST"),
 150                     "ROLE_PERMISSION": "verticalscale:id:",
 151                     "<ID>": {
 152                         "METHODS": ("GET",),
 153                         "ROLE_PERMISSION": "verticalscale:id:id:",
 154                     },
 155                 },
 156             },
 157         }
 158     },
 159 }
 160
 161
 162 class RoException(Exception):
 163     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 164         Exception.__init__(self, message)
 165         self.http_code = http_code
 166
 167
 168 class AuthException(RoException):
 169     pass
 170
 171
 172 class Authenticator:
 173     def __init__(self, valid_url_methods, valid_query_string):
 174         self.valid_url_methods = valid_url_methods
 175         self.valid_query_string = valid_query_string
 176
 177     def authorize(self, *args, **kwargs):
 178         return {"token": "ok", "id": "ok"}
 179
 180     def new_token(self, token_info, indata, remote):
 181         return {"token": "ok", "id": "ok", "remote": remote}
 182
 183     def del_token(self, token_id):
 184         pass
 185
 186     def start(self, engine_config):
 187         pass
 188
 189
 190 class Server(object):
 191     instance = 0
 192     # to decode bytes to str
 193     reader = getreader("utf-8")
 194
 195     def __init__(self):
 196         self.instance += 1
 197         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 198         self.ns = Ns()
 199         self.map_operation = {
 200             "token:post": self.new_token,
 201             "token:id:delete": self.del_token,
 202             "deploy:get": self.ns.get_deploy,
 203             "deploy:id:get": self.ns.get_actions,
 204             "deploy:id:post": self.ns.deploy,
 205             "deploy:id:delete": self.ns.delete,
 206             "deploy:id:id:get": self.ns.status,
 207             "deploy:id:id:cancel:post": self.ns.cancel,
 208             "rebuild:id:post": self.ns.rebuild_start_stop,
 209             "start:id:post": self.ns.rebuild_start_stop,
 210             "stop:id:post": self.ns.rebuild_start_stop,
 211             "recreate:id:post": self.ns.recreate,
 212             "recreate:id:id:get": self.ns.recreate_status,
 213             "migrate:id:post": self.ns.migrate,
 214             "verticalscale:id:post": self.ns.verticalscale,
 215         }
 216
 217     def _format_in(self, kwargs):
 218         error_text = ""
 219         try:
 220             indata = None
 221
 222             if cherrypy.request.body.length:
 223                 error_text = "Invalid input format "
 224
 225                 if "Content-Type" in cherrypy.request.headers:
 226                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 227                         error_text = "Invalid json format "
 228                         indata = json.load(self.reader(cherrypy.request.body))
 229                         cherrypy.request.headers.pop("Content-File-MD5", None)
 230                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 231                         error_text = "Invalid yaml format "
 232                         indata = yaml.safe_load(cherrypy.request.body)
 233                         cherrypy.request.headers.pop("Content-File-MD5", None)
 234                     elif (
 235                         "application/binary" in cherrypy.request.headers["Content-Type"]
 236                         or "application/gzip"
 237                         in cherrypy.request.headers["Content-Type"]
 238                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 239                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 240                     ):
 241                         indata = cherrypy.request.body  # .read()
 242                     elif (
 243                         "multipart/form-data"
 244                         in cherrypy.request.headers["Content-Type"]
 245                     ):
 246                         if "descriptor_file" in kwargs:
 247                             filecontent = kwargs.pop("descriptor_file")
 248
 249                             if not filecontent.file:
 250                                 raise RoException(
 251                                     "empty file or content", HTTPStatus.BAD_REQUEST
 252                                 )
 253
 254                             indata = filecontent.file  # .read()
 255
 256                             if filecontent.content_type.value:
 257                                 cherrypy.request.headers[
 258                                     "Content-Type"
 259                                 ] = filecontent.content_type.value
 260                     else:
 261                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 262                         #                          "Only 'Content-Type' of type 'application/json' or
 263                         # 'application/yaml' for input format are available")
 264                         error_text = "Invalid yaml format "
 265                         indata = yaml.safe_load(cherrypy.request.body)
 266                         cherrypy.request.headers.pop("Content-File-MD5", None)
 267                 else:
 268                     error_text = "Invalid yaml format "
 269                     indata = yaml.safe_load(cherrypy.request.body)
 270                     cherrypy.request.headers.pop("Content-File-MD5", None)
 271
 272             if not indata:
 273                 indata = {}
 274
 275             format_yaml = False
 276             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 277                 format_yaml = True
 278
 279             for k, v in kwargs.items():
 280                 if isinstance(v, str):
 281                     if v == "":
 282                         kwargs[k] = None
 283                     elif format_yaml:
 284                         try:
 285                             kwargs[k] = yaml.safe_load(v)
 286                         except Exception as yaml_error:
 287                             logging.exception(
 288                                 f"{yaml_error} occured while parsing the yaml"
 289                             )
 290                     elif (
 291                         k.endswith(".gt")
 292                         or k.endswith(".lt")
 293                         or k.endswith(".gte")
 294                         or k.endswith(".lte")
 295                     ):
 296                         try:
 297                             kwargs[k] = int(v)
 298                         except Exception:
 299                             try:
 300                                 kwargs[k] = float(v)
 301                             except Exception as keyword_error:
 302                                 logging.exception(
 303                                     f"{keyword_error} occured while getting the keyword arguments"
 304                                 )
 305                     elif v.find(",") > 0:
 306                         kwargs[k] = v.split(",")
 307                 elif isinstance(v, (list, tuple)):
 308                     for index in range(0, len(v)):
 309                         if v[index] == "":
 310                             v[index] = None
 311                         elif format_yaml:
 312                             try:
 313                                 v[index] = yaml.safe_load(v[index])
 314                             except Exception as error:
 315                                 logging.exception(
 316                                     f"{error} occured while parsing the yaml"
 317                                 )
 318
 319             return indata
 320         except (ValueError, yaml.YAMLError) as exc:
 321             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 322         except KeyError as exc:
 323             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 324         except Exception as exc:
 325             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 326
 327     @staticmethod
 328     def _format_out(data, token_info=None, _format=None):
 329         """
 330         return string of dictionary data according to requested json, yaml, xml. By default json
 331         :param data: response to be sent. Can be a dict, text or file
 332         :param token_info: Contains among other username and project
 333         :param _format: The format to be set as Content-Type if data is a file
 334         :return: None
 335         """
 336         accept = cherrypy.request.headers.get("Accept")
 337
 338         if data is None:
 339             if accept and "text/html" in accept:
 340                 return html.format(
 341                     data, cherrypy.request, cherrypy.response, token_info
 342                 )
 343
 344             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 345             return
 346         elif hasattr(data, "read"):  # file object
 347             if _format:
 348                 cherrypy.response.headers["Content-Type"] = _format
 349             elif "b" in data.mode:  # binariy asssumig zip
 350                 cherrypy.response.headers["Content-Type"] = "application/zip"
 351             else:
 352                 cherrypy.response.headers["Content-Type"] = "text/plain"
 353
 354             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 355             return data
 356
 357         if accept:
 358             if "application/json" in accept:
 359                 cherrypy.response.headers[
 360                     "Content-Type"
 361                 ] = "application/json; charset=utf-8"
 362                 a = json.dumps(data, indent=4) + "\n"
 363
 364                 return a.encode("utf8")
 365             elif "text/html" in accept:
 366                 return html.format(
 367                     data, cherrypy.request, cherrypy.response, token_info
 368                 )
 369             elif (
 370                 "application/yaml" in accept
 371                 or "*/*" in accept
 372                 or "text/plain" in accept
 373             ):
 374                 pass
 375             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 376             elif cherrypy.response.status >= 400:
 377                 raise cherrypy.HTTPError(
 378                     HTTPStatus.NOT_ACCEPTABLE.value,
 379                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 380                     "for output format are available",
 381                 )
 382
 383         cherrypy.response.headers["Content-Type"] = "application/yaml"
 384
 385         return yaml.safe_dump(
 386             data,
 387             explicit_start=True,
 388             indent=4,
 389             default_flow_style=False,
 390             tags=False,
 391             encoding="utf-8",
 392             allow_unicode=True,
 393         )  # , canonical=True, default_style='"'
 394
 395     @cherrypy.expose
 396     def index(self, *args, **kwargs):
 397         token_info = None
 398
 399         try:
 400             if cherrypy.request.method == "GET":
 401                 token_info = self.authenticator.authorize()
 402                 outdata = token_info  # Home page
 403             else:
 404                 raise cherrypy.HTTPError(
 405                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 406                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 407                 )
 408
 409             return self._format_out(outdata, token_info)
 410         except (NsException, AuthException) as e:
 411             # cherrypy.log("index Exception {}".format(e))
 412             cherrypy.response.status = e.http_code.value
 413
 414             return self._format_out("Welcome to OSM!", token_info)
 415
 416     @cherrypy.expose
 417     def version(self, *args, **kwargs):
 418         # TODO consider to remove and provide version using the static version file
 419         try:
 420             if cherrypy.request.method != "GET":
 421                 raise RoException(
 422                     "Only method GET is allowed",
 423                     HTTPStatus.METHOD_NOT_ALLOWED,
 424                 )
 425             elif args or kwargs:
 426                 raise RoException(
 427                     "Invalid URL or query string for version",
 428                     HTTPStatus.METHOD_NOT_ALLOWED,
 429                 )
 430
 431             # TODO include version of other modules, pick up from some kafka admin message
 432             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 433
 434             return self._format_out(osm_ng_ro_version)
 435         except RoException as e:
 436             cherrypy.response.status = e.http_code.value
 437             problem_details = {
 438                 "code": e.http_code.name,
 439                 "status": e.http_code.value,
 440                 "detail": str(e),
 441             }
 442
 443             return self._format_out(problem_details, None)
 444
 445     def new_token(self, engine_session, indata, *args, **kwargs):
 446         token_info = None
 447
 448         try:
 449             token_info = self.authenticator.authorize()
 450         except Exception:
 451             token_info = None
 452
 453         if kwargs:
 454             indata.update(kwargs)
 455
 456         # This is needed to log the user when authentication fails
 457         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 458         token_info = self.authenticator.new_token(
 459             token_info, indata, cherrypy.request.remote
 460         )
 461         cherrypy.session["Authorization"] = token_info["id"]
 462         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 463         # for logging
 464
 465         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 466         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 467
 468         return token_info, token_info["id"], True
 469
 470     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 471         token_id = _id
 472
 473         if not token_id and "id" in kwargs:
 474             token_id = kwargs["id"]
 475         elif not token_id:
 476             token_info = self.authenticator.authorize()
 477             # for logging
 478             token_id = token_info["id"]
 479
 480         self.authenticator.del_token(token_id)
 481         token_info = None
 482         cherrypy.session["Authorization"] = "logout"
 483         # cherrypy.response.cookie["Authorization"] = token_id
 484         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 485
 486         return None, None, True
 487
 488     @cherrypy.expose
 489     def test(self, *args, **kwargs):
 490         if not cherrypy.config.get("server.enable_test") or (
 491             isinstance(cherrypy.config["server.enable_test"], str)
 492             and cherrypy.config["server.enable_test"].lower() == "false"
 493         ):
 494             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 495
 496             return "test URL is disabled"
 497
 498         thread_info = None
 499
 500         if args and args[0] == "help":
 501             return (
 502                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 503                 "sleep/<time>\nmessage/topic\n</pre></html>"
 504             )
 505         elif args and args[0] == "init":
 506             try:
 507                 # self.ns.load_dbase(cherrypy.request.app.config)
 508                 self.ns.create_admin()
 509
 510                 return "Done. User 'admin', password 'admin' created"
 511             except Exception:
 512                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 513
 514                 return self._format_out("Database already initialized")
 515         elif args and args[0] == "file":
 516             return cherrypy.lib.static.serve_file(
 517                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 518                 "text/plain",
 519                 "attachment",
 520             )
 521         elif args and args[0] == "file2":
 522             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 523             f = open(f_path, "r")
 524             cherrypy.response.headers["Content-type"] = "text/plain"
 525             return f
 526
 527         elif len(args) == 2 and args[0] == "db-clear":
 528             deleted_info = self.ns.db.del_list(args[1], kwargs)
 529             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 530         elif len(args) and args[0] == "fs-clear":
 531             if len(args) >= 2:
 532                 folders = (args[1],)
 533             else:
 534                 folders = self.ns.fs.dir_ls(".")
 535
 536             for folder in folders:
 537                 self.ns.fs.file_delete(folder)
 538
 539             return ",".join(folders) + " folders deleted\n"
 540         elif args and args[0] == "login":
 541             if not cherrypy.request.headers.get("Authorization"):
 542                 cherrypy.response.headers[
 543                     "WWW-Authenticate"
 544                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 545                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 546         elif args and args[0] == "login2":
 547             if not cherrypy.request.headers.get("Authorization"):
 548                 cherrypy.response.headers[
 549                     "WWW-Authenticate"
 550                 ] = 'Bearer realm="Access to OSM site"'
 551                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 552         elif args and args[0] == "sleep":
 553             sleep_time = 5
 554
 555             try:
 556                 sleep_time = int(args[1])
 557             except Exception:
 558                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 559                 return self._format_out("Database already initialized")
 560
 561             thread_info = cherrypy.thread_data
 562             print(thread_info)
 563             time.sleep(sleep_time)
 564             # thread_info
 565         elif len(args) >= 2 and args[0] == "message":
 566             main_topic = args[1]
 567             return_text = "<html><pre>{} ->\n".format(main_topic)
 568
 569             try:
 570                 if cherrypy.request.method == "POST":
 571                     to_send = yaml.safe_load(cherrypy.request.body)
 572                     for k, v in to_send.items():
 573                         self.ns.msg.write(main_topic, k, v)
 574                         return_text += "  {}: {}\n".format(k, v)
 575                 elif cherrypy.request.method == "GET":
 576                     for k, v in kwargs.items():
 577                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 578                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 579             except Exception as e:
 580                 return_text += "Error: " + str(e)
 581
 582             return_text += "</pre></html>\n"
 583
 584             return return_text
 585
 586         return_text = (
 587             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 588             + "  kwargs: {}\n".format(kwargs)
 589             + "  headers: {}\n".format(cherrypy.request.headers)
 590             + "  path_info: {}\n".format(cherrypy.request.path_info)
 591             + "  query_string: {}\n".format(cherrypy.request.query_string)
 592             + "  session: {}\n".format(cherrypy.session)
 593             + "  cookie: {}\n".format(cherrypy.request.cookie)
 594             + "  method: {}\n".format(cherrypy.request.method)
 595             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 596             + "  body:\n"
 597         )
 598         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 599
 600         if cherrypy.request.body.length:
 601             return_text += "    content: {}\n".format(
 602                 str(
 603                     cherrypy.request.body.read(
 604                         int(cherrypy.request.headers.get("Content-Length", 0))
 605                     )
 606                 )
 607             )
 608
 609         if thread_info:
 610             return_text += "thread: {}\n".format(thread_info)
 611
 612         return_text += "</pre></html>"
 613
 614         return return_text
 615
 616     @staticmethod
 617     def _check_valid_url_method(method, *args):
 618         if len(args) < 3:
 619             raise RoException(
 620                 "URL must contain at least 'main_topic/version/topic'",
 621                 HTTPStatus.METHOD_NOT_ALLOWED,
 622             )
 623
 624         reference = valid_url_methods
 625         for arg in args:
 626             if arg is None:
 627                 break
 628
 629             if not isinstance(reference, dict):
 630                 raise RoException(
 631                     "URL contains unexpected extra items '{}'".format(arg),
 632                     HTTPStatus.METHOD_NOT_ALLOWED,
 633                 )
 634
 635             if arg in reference:
 636                 reference = reference[arg]
 637             elif "<ID>" in reference:
 638                 reference = reference["<ID>"]
 639             elif "*" in reference:
 640                 # reference = reference["*"]
 641                 break
 642             else:
 643                 raise RoException(
 644                     "Unexpected URL item {}".format(arg),
 645                     HTTPStatus.METHOD_NOT_ALLOWED,
 646                 )
 647
 648         if "TODO" in reference and method in reference["TODO"]:
 649             raise RoException(
 650                 "Method {} not supported yet for this URL".format(method),
 651                 HTTPStatus.NOT_IMPLEMENTED,
 652             )
 653         elif "METHODS" not in reference or method not in reference["METHODS"]:
 654             raise RoException(
 655                 "Method {} not supported for this URL".format(method),
 656                 HTTPStatus.METHOD_NOT_ALLOWED,
 657             )
 658
 659         return reference["ROLE_PERMISSION"] + method.lower()
 660
 661     @staticmethod
 662     def _set_location_header(main_topic, version, topic, id):
 663         """
 664         Insert response header Location with the URL of created item base on URL params
 665         :param main_topic:
 666         :param version:
 667         :param topic:
 668         :param id:
 669         :return: None
 670         """
 671         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 672         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 673             main_topic, version, topic, id
 674         )
 675
 676         return
 677
 678     @cherrypy.expose
 679     def default(
 680         self,
 681         main_topic=None,
 682         version=None,
 683         topic=None,
 684         _id=None,
 685         _id2=None,
 686         *args,
 687         **kwargs,
 688     ):
 689         token_info = None
 690         outdata = None
 691         _format = None
 692         method = "DONE"
 693         rollback = []
 694         engine_session = None
 695
 696         try:
 697             if not main_topic or not version or not topic:
 698                 raise RoException(
 699                     "URL must contain at least 'main_topic/version/topic'",
 700                     HTTPStatus.METHOD_NOT_ALLOWED,
 701                 )
 702
 703             if main_topic not in (
 704                 "admin",
 705                 "ns",
 706             ):
 707                 raise RoException(
 708                     "URL main_topic '{}' not supported".format(main_topic),
 709                     HTTPStatus.METHOD_NOT_ALLOWED,
 710                 )
 711
 712             if version != "v1":
 713                 raise RoException(
 714                     "URL version '{}' not supported".format(version),
 715                     HTTPStatus.METHOD_NOT_ALLOWED,
 716                 )
 717
 718             if (
 719                 kwargs
 720                 and "METHOD" in kwargs
 721                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 722             ):
 723                 method = kwargs.pop("METHOD")
 724             else:
 725                 method = cherrypy.request.method
 726
 727             role_permission = self._check_valid_url_method(
 728                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 729             )
 730             # skip token validation if requesting a token
 731             indata = self._format_in(kwargs)
 732
 733             if main_topic != "admin" or topic != "tokens":
 734                 token_info = self.authenticator.authorize(role_permission, _id)
 735
 736             outdata, created_id, done = self.map_operation[role_permission](
 737                 engine_session, indata, version, _id, _id2, *args, *kwargs
 738             )
 739
 740             if created_id:
 741                 self._set_location_header(main_topic, version, topic, _id)
 742
 743             cherrypy.response.status = (
 744                 HTTPStatus.ACCEPTED.value
 745                 if not done
 746                 else HTTPStatus.OK.value
 747                 if outdata is not None
 748                 else HTTPStatus.NO_CONTENT.value
 749             )
 750
 751             return self._format_out(outdata, token_info, _format)
 752         except Exception as e:
 753             if isinstance(
 754                 e,
 755                 (
 756                     RoException,
 757                     NsException,
 758                     DbException,
 759                     FsException,
 760                     MsgException,
 761                     AuthException,
 762                     ValidationError,
 763                 ),
 764             ):
 765                 http_code_value = cherrypy.response.status = e.http_code.value
 766                 http_code_name = e.http_code.name
 767                 cherrypy.log("Exception {}".format(e))
 768             else:
 769                 http_code_value = (
 770                     cherrypy.response.status
 771                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 772                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 773                 http_code_name = HTTPStatus.BAD_REQUEST.name
 774
 775             if hasattr(outdata, "close"):  # is an open file
 776                 outdata.close()
 777
 778             error_text = str(e)
 779             rollback.reverse()
 780
 781             for rollback_item in rollback:
 782                 try:
 783                     if rollback_item.get("operation") == "set":
 784                         self.ns.db.set_one(
 785                             rollback_item["topic"],
 786                             {"_id": rollback_item["_id"]},
 787                             rollback_item["content"],
 788                             fail_on_empty=False,
 789                         )
 790                     else:
 791                         self.ns.db.del_one(
 792                             rollback_item["topic"],
 793                             {"_id": rollback_item["_id"]},
 794                             fail_on_empty=False,
 795                         )
 796                 except Exception as e2:
 797                     rollback_error_text = "Rollback Exception {}: {}".format(
 798                         rollback_item, e2
 799                     )
 800                     cherrypy.log(rollback_error_text)
 801                     error_text += ". " + rollback_error_text
 802
 803             # if isinstance(e, MsgException):
 804             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 805             #         engine_topic[:-1], method, error_text)
 806             problem_details = {
 807                 "code": http_code_name,
 808                 "status": http_code_value,
 809                 "detail": error_text,
 810             }
 811
 812             return self._format_out(problem_details, token_info)
 813             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 814         finally:
 815             if token_info:
 816                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 817                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 818                         if outdata.get(logging_id):
 819                             cherrypy.request.login += ";{}={}".format(
 820                                 logging_id, outdata[logging_id][:36]
 821                             )
 822
 823
 824 def _start_service():
 825     """
 826     Callback function called when cherrypy.engine starts
 827     Override configuration with env variables
 828     Set database, storage, message configuration
 829     Init database with admin/admin user password
 830     """
 831     global ro_server, vim_admin_thread
 832     # global vim_threads
 833     cherrypy.log.error("Starting osm_ng_ro")
 834     # update general cherrypy configuration
 835     update_dict = {}
 836     engine_config = cherrypy.tree.apps["/ro"].config
 837
 838     for k, v in environ.items():
 839         if not k.startswith("OSMRO_"):
 840             continue
 841
 842         k1, _, k2 = k[6:].lower().partition("_")
 843
 844         if not k2:
 845             continue
 846
 847         try:
 848             if k1 in ("server", "test", "auth", "log"):
 849                 # update [global] configuration
 850                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 851             elif k1 == "static":
 852                 # update [/static] configuration
 853                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 854             elif k1 == "tools":
 855                 # update [/] configuration
 856                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 857             elif k1 in ("message", "database", "storage", "authentication", "period"):
 858                 engine_config[k1][k2] = yaml.safe_load(v)
 859
 860         except Exception as e:
 861             raise RoException("Cannot load env '{}': {}".format(k, e))
 862
 863     if update_dict:
 864         cherrypy.config.update(update_dict)
 865         engine_config["global"].update(update_dict)
 866
 867     # logging cherrypy
 868     log_format_simple = (
 869         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 870     )
 871     log_formatter_simple = logging.Formatter(
 872         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 873     )
 874     logger_server = logging.getLogger("cherrypy.error")
 875     logger_access = logging.getLogger("cherrypy.access")
 876     logger_cherry = logging.getLogger("cherrypy")
 877     logger = logging.getLogger("ro")
 878
 879     if "log.file" in engine_config["global"]:
 880         file_handler = logging.handlers.RotatingFileHandler(
 881             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 882         )
 883         file_handler.setFormatter(log_formatter_simple)
 884         logger_cherry.addHandler(file_handler)
 885         logger.addHandler(file_handler)
 886
 887     # log always to standard output
 888     for format_, logger in {
 889         "ro.server %(filename)s:%(lineno)s": logger_server,
 890         "ro.access %(filename)s:%(lineno)s": logger_access,
 891         "%(name)s %(filename)s:%(lineno)s": logger,
 892     }.items():
 893         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 894         log_formatter_cherry = logging.Formatter(
 895             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 896         )
 897         str_handler = logging.StreamHandler()
 898         str_handler.setFormatter(log_formatter_cherry)
 899         logger.addHandler(str_handler)
 900
 901     if engine_config["global"].get("log.level"):
 902         logger_cherry.setLevel(engine_config["global"]["log.level"])
 903         logger.setLevel(engine_config["global"]["log.level"])
 904
 905     # logging other modules
 906     for k1, logname in {
 907         "message": "ro.msg",
 908         "database": "ro.db",
 909         "storage": "ro.fs",
 910     }.items():
 911         engine_config[k1]["logger_name"] = logname
 912         logger_module = logging.getLogger(logname)
 913
 914         if "logfile" in engine_config[k1]:
 915             file_handler = logging.handlers.RotatingFileHandler(
 916                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 917             )
 918             file_handler.setFormatter(log_formatter_simple)
 919             logger_module.addHandler(file_handler)
 920
 921         if "loglevel" in engine_config[k1]:
 922             logger_module.setLevel(engine_config[k1]["loglevel"])
 923     # TODO add more entries, e.g.: storage
 924
 925     engine_config["assignment"] = {}
 926     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 927     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 928     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 929     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 930
 931     # # start subscriptions thread:
 932     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 933     vim_admin_thread.start()
 934     start_monitoring(config=engine_config)
 935
 936     # # Do not capture except SubscriptionException
 937
 938     # backend = engine_config["authentication"]["backend"]
 939     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 940     #                    .format(ro_version, ro_version_date, backend))
 941
 942
 943 def _stop_service():
 944     """
 945     Callback function called when cherrypy.engine stops
 946     TODO: Ending database connections.
 947     """
 948     global vim_admin_thread
 949
 950     # terminate vim_admin_thread
 951     if vim_admin_thread:
 952         vim_admin_thread.terminate()
 953     stop_monitoring()
 954     vim_admin_thread = None
 955     cherrypy.tree.apps["/ro"].root.ns.stop()
 956     cherrypy.log.error("Stopping osm_ng_ro")
 957
 958
 959 def ro_main(config_file):
 960     global ro_server
 961
 962     ro_server = Server()
 963     cherrypy.engine.subscribe("start", _start_service)
 964     cherrypy.engine.subscribe("stop", _stop_service)
 965     cherrypy.quickstart(ro_server, "/ro", config_file)
 966
 967
 968 def usage():
 969     print(
 970         """Usage: {} [options]
 971         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 972         -h|--help: shows this help
 973         """.format(
 974             sys.argv[0]
 975         )
 976     )
 977     # --log-socket-host HOST: send logs to this host")
 978     # --log-socket-port PORT: send logs using this port (default: 9022)")
 979
 980
 981 if __name__ == "__main__":
 982     try:
 983         # load parameters and configuration
 984         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 985         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 986         config_file = None
 987
 988         for o, a in opts:
 989             if o in ("-h", "--help"):
 990                 usage()
 991                 sys.exit()
 992             elif o in ("-c", "--config"):
 993                 config_file = a
 994             else:
 995                 raise ValueError("Unhandled option")
 996
 997         if config_file:
 998             if not path.isfile(config_file):
 999                 print(
1000                     "configuration file '{}' that not exist".format(config_file),
1001                     file=sys.stderr,
1002                 )
1003                 exit(1)
1004         else:
1005             for config_file in (
1006                 path.dirname(__file__) + "/ro.cfg",
1007                 "./ro.cfg",
1008                 "/etc/osm/ro.cfg",
1009             ):
1010                 if path.isfile(config_file):
1011                     break
1012             else:
1013                 print(
1014                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1015                     file=sys.stderr,
1016                 )
1017                 exit(1)
1018
1019         ro_main(config_file)
1020     except KeyboardInterrupt:
1021         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1022     except getopt.GetoptError as e:
1023         print(str(e), file=sys.stderr)
1024         # usage()
1025         exit(1)