NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "rebuild": {
  87                 "METHODS": ("POST",),
  88                 "ROLE_PERMISSION": "rebuild:",
  89                 "<ID>": {
  90                     "METHODS": ("POST",),
  91                     "ROLE_PERMISSION": "rebuild:id:",
  92                 },
  93             },
  94             "start": {
  95                 "METHODS": ("POST",),
  96                 "ROLE_PERMISSION": "start:",
  97                 "<ID>": {
  98                     "METHODS": ("POST",),
  99                     "ROLE_PERMISSION": "start:id:",
 100                 },
 101             },
 102             "stop": {
 103                 "METHODS": ("POST",),
 104                 "ROLE_PERMISSION": "stop:",
 105                 "<ID>": {
 106                     "METHODS": ("POST",),
 107                     "ROLE_PERMISSION": "stop:id:",
 108                 },
 109             },
 110             "deploy": {
 111                 "METHODS": ("GET",),
 112                 "ROLE_PERMISSION": "deploy:",
 113                 "<ID>": {
 114                     "METHODS": ("GET", "POST", "DELETE"),
 115                     "ROLE_PERMISSION": "deploy:id:",
 116                     "<ID>": {
 117                         "METHODS": ("GET",),
 118                         "ROLE_PERMISSION": "deploy:id:id:",
 119                         "cancel": {
 120                             "METHODS": ("POST",),
 121                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 122                         },
 123                     },
 124                 },
 125             },
 126             "recreate": {
 127                 "<ID>": {
 128                     "METHODS": ("POST"),
 129                     "ROLE_PERMISSION": "recreate:id:",
 130                     "<ID>": {
 131                         "METHODS": ("GET",),
 132                         "ROLE_PERMISSION": "recreate:id:id:",
 133                     },
 134                 },
 135             },
 136             "migrate": {
 137                 "<ID>": {
 138                     "METHODS": ("POST"),
 139                     "ROLE_PERMISSION": "migrate:id:",
 140                     "<ID>": {
 141                         "METHODS": ("GET",),
 142                         "ROLE_PERMISSION": "migrate:id:id:",
 143                     },
 144                 },
 145             },
 146             "verticalscale": {
 147                 "<ID>": {
 148                     "METHODS": ("POST"),
 149                     "ROLE_PERMISSION": "verticalscale:id:",
 150                     "<ID>": {
 151                         "METHODS": ("GET",),
 152                         "ROLE_PERMISSION": "verticalscale:id:id:",
 153                     },
 154                 },
 155             },
 156         }
 157     },
 158 }
 159
 160
 161 class RoException(Exception):
 162     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 163         Exception.__init__(self, message)
 164         self.http_code = http_code
 165
 166
 167 class AuthException(RoException):
 168     pass
 169
 170
 171 class Authenticator:
 172     def __init__(self, valid_url_methods, valid_query_string):
 173         self.valid_url_methods = valid_url_methods
 174         self.valid_query_string = valid_query_string
 175
 176     def authorize(self, *args, **kwargs):
 177         return {"token": "ok", "id": "ok"}
 178
 179     def new_token(self, token_info, indata, remote):
 180         return {"token": "ok", "id": "ok", "remote": remote}
 181
 182     def del_token(self, token_id):
 183         pass
 184
 185     def start(self, engine_config):
 186         pass
 187
 188
 189 class Server(object):
 190     instance = 0
 191     # to decode bytes to str
 192     reader = getreader("utf-8")
 193
 194     def __init__(self):
 195         self.instance += 1
 196         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 197         self.ns = Ns()
 198         self.map_operation = {
 199             "token:post": self.new_token,
 200             "token:id:delete": self.del_token,
 201             "deploy:get": self.ns.get_deploy,
 202             "deploy:id:get": self.ns.get_actions,
 203             "deploy:id:post": self.ns.deploy,
 204             "deploy:id:delete": self.ns.delete,
 205             "deploy:id:id:get": self.ns.status,
 206             "deploy:id:id:cancel:post": self.ns.cancel,
 207             "rebuild:id:post": self.ns.rebuild_start_stop,
 208             "start:id:post": self.ns.rebuild_start_stop,
 209             "stop:id:post": self.ns.rebuild_start_stop,
 210             "recreate:id:post": self.ns.recreate,
 211             "recreate:id:id:get": self.ns.recreate_status,
 212             "migrate:id:post": self.ns.migrate,
 213             "verticalscale:id:post": self.ns.verticalscale,
 214         }
 215
 216     def _format_in(self, kwargs):
 217         try:
 218             indata = None
 219
 220             if cherrypy.request.body.length:
 221                 error_text = "Invalid input format "
 222
 223                 if "Content-Type" in cherrypy.request.headers:
 224                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 225                         error_text = "Invalid json format "
 226                         indata = json.load(self.reader(cherrypy.request.body))
 227                         cherrypy.request.headers.pop("Content-File-MD5", None)
 228                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 229                         error_text = "Invalid yaml format "
 230                         indata = yaml.safe_load(cherrypy.request.body)
 231                         cherrypy.request.headers.pop("Content-File-MD5", None)
 232                     elif (
 233                         "application/binary" in cherrypy.request.headers["Content-Type"]
 234                         or "application/gzip"
 235                         in cherrypy.request.headers["Content-Type"]
 236                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 237                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 238                     ):
 239                         indata = cherrypy.request.body  # .read()
 240                     elif (
 241                         "multipart/form-data"
 242                         in cherrypy.request.headers["Content-Type"]
 243                     ):
 244                         if "descriptor_file" in kwargs:
 245                             filecontent = kwargs.pop("descriptor_file")
 246
 247                             if not filecontent.file:
 248                                 raise RoException(
 249                                     "empty file or content", HTTPStatus.BAD_REQUEST
 250                                 )
 251
 252                             indata = filecontent.file  # .read()
 253
 254                             if filecontent.content_type.value:
 255                                 cherrypy.request.headers[
 256                                     "Content-Type"
 257                                 ] = filecontent.content_type.value
 258                     else:
 259                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 260                         #                          "Only 'Content-Type' of type 'application/json' or
 261                         # 'application/yaml' for input format are available")
 262                         error_text = "Invalid yaml format "
 263                         indata = yaml.safe_load(cherrypy.request.body)
 264                         cherrypy.request.headers.pop("Content-File-MD5", None)
 265                 else:
 266                     error_text = "Invalid yaml format "
 267                     indata = yaml.safe_load(cherrypy.request.body)
 268                     cherrypy.request.headers.pop("Content-File-MD5", None)
 269
 270             if not indata:
 271                 indata = {}
 272
 273             format_yaml = False
 274             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 275                 format_yaml = True
 276
 277             for k, v in kwargs.items():
 278                 if isinstance(v, str):
 279                     if v == "":
 280                         kwargs[k] = None
 281                     elif format_yaml:
 282                         try:
 283                             kwargs[k] = yaml.safe_load(v)
 284                         except Exception as yaml_error:
 285                             logging.exception(
 286                                 f"{yaml_error} occured while parsing the yaml"
 287                             )
 288                     elif (
 289                         k.endswith(".gt")
 290                         or k.endswith(".lt")
 291                         or k.endswith(".gte")
 292                         or k.endswith(".lte")
 293                     ):
 294                         try:
 295                             kwargs[k] = int(v)
 296                         except Exception:
 297                             try:
 298                                 kwargs[k] = float(v)
 299                             except Exception as keyword_error:
 300                                 logging.exception(
 301                                     f"{keyword_error} occured while getting the keyword arguments"
 302                                 )
 303                     elif v.find(",") > 0:
 304                         kwargs[k] = v.split(",")
 305                 elif isinstance(v, (list, tuple)):
 306                     for index in range(0, len(v)):
 307                         if v[index] == "":
 308                             v[index] = None
 309                         elif format_yaml:
 310                             try:
 311                                 v[index] = yaml.safe_load(v[index])
 312                             except Exception as error:
 313                                 logging.exception(
 314                                     f"{error} occured while parsing the yaml"
 315                                 )
 316
 317             return indata
 318         except (ValueError, yaml.YAMLError) as exc:
 319             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 320         except KeyError as exc:
 321             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 322         except Exception as exc:
 323             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 324
 325     @staticmethod
 326     def _format_out(data, token_info=None, _format=None):
 327         """
 328         return string of dictionary data according to requested json, yaml, xml. By default json
 329         :param data: response to be sent. Can be a dict, text or file
 330         :param token_info: Contains among other username and project
 331         :param _format: The format to be set as Content-Type if data is a file
 332         :return: None
 333         """
 334         accept = cherrypy.request.headers.get("Accept")
 335
 336         if data is None:
 337             if accept and "text/html" in accept:
 338                 return html.format(
 339                     data, cherrypy.request, cherrypy.response, token_info
 340                 )
 341
 342             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 343             return
 344         elif hasattr(data, "read"):  # file object
 345             if _format:
 346                 cherrypy.response.headers["Content-Type"] = _format
 347             elif "b" in data.mode:  # binariy asssumig zip
 348                 cherrypy.response.headers["Content-Type"] = "application/zip"
 349             else:
 350                 cherrypy.response.headers["Content-Type"] = "text/plain"
 351
 352             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 353             return data
 354
 355         if accept:
 356             if "application/json" in accept:
 357                 cherrypy.response.headers[
 358                     "Content-Type"
 359                 ] = "application/json; charset=utf-8"
 360                 a = json.dumps(data, indent=4) + "\n"
 361
 362                 return a.encode("utf8")
 363             elif "text/html" in accept:
 364                 return html.format(
 365                     data, cherrypy.request, cherrypy.response, token_info
 366                 )
 367             elif (
 368                 "application/yaml" in accept
 369                 or "*/*" in accept
 370                 or "text/plain" in accept
 371             ):
 372                 pass
 373             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 374             elif cherrypy.response.status >= 400:
 375                 raise cherrypy.HTTPError(
 376                     HTTPStatus.NOT_ACCEPTABLE.value,
 377                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 378                     "for output format are available",
 379                 )
 380
 381         cherrypy.response.headers["Content-Type"] = "application/yaml"
 382
 383         return yaml.safe_dump(
 384             data,
 385             explicit_start=True,
 386             indent=4,
 387             default_flow_style=False,
 388             tags=False,
 389             encoding="utf-8",
 390             allow_unicode=True,
 391         )  # , canonical=True, default_style='"'
 392
 393     @cherrypy.expose
 394     def index(self, *args, **kwargs):
 395         token_info = None
 396
 397         try:
 398             if cherrypy.request.method == "GET":
 399                 token_info = self.authenticator.authorize()
 400                 outdata = token_info  # Home page
 401             else:
 402                 raise cherrypy.HTTPError(
 403                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 404                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 405                 )
 406
 407             return self._format_out(outdata, token_info)
 408         except (NsException, AuthException) as e:
 409             # cherrypy.log("index Exception {}".format(e))
 410             cherrypy.response.status = e.http_code.value
 411
 412             return self._format_out("Welcome to OSM!", token_info)
 413
 414     @cherrypy.expose
 415     def version(self, *args, **kwargs):
 416         # TODO consider to remove and provide version using the static version file
 417         try:
 418             if cherrypy.request.method != "GET":
 419                 raise RoException(
 420                     "Only method GET is allowed",
 421                     HTTPStatus.METHOD_NOT_ALLOWED,
 422                 )
 423             elif args or kwargs:
 424                 raise RoException(
 425                     "Invalid URL or query string for version",
 426                     HTTPStatus.METHOD_NOT_ALLOWED,
 427                 )
 428
 429             # TODO include version of other modules, pick up from some kafka admin message
 430             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 431
 432             return self._format_out(osm_ng_ro_version)
 433         except RoException as e:
 434             cherrypy.response.status = e.http_code.value
 435             problem_details = {
 436                 "code": e.http_code.name,
 437                 "status": e.http_code.value,
 438                 "detail": str(e),
 439             }
 440
 441             return self._format_out(problem_details, None)
 442
 443     def new_token(self, engine_session, indata, *args, **kwargs):
 444         token_info = None
 445
 446         try:
 447             token_info = self.authenticator.authorize()
 448         except Exception:
 449             token_info = None
 450
 451         if kwargs:
 452             indata.update(kwargs)
 453
 454         # This is needed to log the user when authentication fails
 455         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 456         token_info = self.authenticator.new_token(
 457             token_info, indata, cherrypy.request.remote
 458         )
 459         cherrypy.session["Authorization"] = token_info["id"]
 460         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 461         # for logging
 462
 463         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 464         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 465
 466         return token_info, token_info["id"], True
 467
 468     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 469         token_id = _id
 470
 471         if not token_id and "id" in kwargs:
 472             token_id = kwargs["id"]
 473         elif not token_id:
 474             token_info = self.authenticator.authorize()
 475             # for logging
 476             token_id = token_info["id"]
 477
 478         self.authenticator.del_token(token_id)
 479         token_info = None
 480         cherrypy.session["Authorization"] = "logout"
 481         # cherrypy.response.cookie["Authorization"] = token_id
 482         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 483
 484         return None, None, True
 485
 486     @cherrypy.expose
 487     def test(self, *args, **kwargs):
 488         if not cherrypy.config.get("server.enable_test") or (
 489             isinstance(cherrypy.config["server.enable_test"], str)
 490             and cherrypy.config["server.enable_test"].lower() == "false"
 491         ):
 492             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 493
 494             return "test URL is disabled"
 495
 496         thread_info = None
 497
 498         if args and args[0] == "help":
 499             return (
 500                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 501                 "sleep/<time>\nmessage/topic\n</pre></html>"
 502             )
 503         elif args and args[0] == "init":
 504             try:
 505                 # self.ns.load_dbase(cherrypy.request.app.config)
 506                 self.ns.create_admin()
 507
 508                 return "Done. User 'admin', password 'admin' created"
 509             except Exception:
 510                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 511
 512                 return self._format_out("Database already initialized")
 513         elif args and args[0] == "file":
 514             return cherrypy.lib.static.serve_file(
 515                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 516                 "text/plain",
 517                 "attachment",
 518             )
 519         elif args and args[0] == "file2":
 520             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 521             f = open(f_path, "r")
 522             cherrypy.response.headers["Content-type"] = "text/plain"
 523             return f
 524
 525         elif len(args) == 2 and args[0] == "db-clear":
 526             deleted_info = self.ns.db.del_list(args[1], kwargs)
 527             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 528         elif len(args) and args[0] == "fs-clear":
 529             if len(args) >= 2:
 530                 folders = (args[1],)
 531             else:
 532                 folders = self.ns.fs.dir_ls(".")
 533
 534             for folder in folders:
 535                 self.ns.fs.file_delete(folder)
 536
 537             return ",".join(folders) + " folders deleted\n"
 538         elif args and args[0] == "login":
 539             if not cherrypy.request.headers.get("Authorization"):
 540                 cherrypy.response.headers[
 541                     "WWW-Authenticate"
 542                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 543                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 544         elif args and args[0] == "login2":
 545             if not cherrypy.request.headers.get("Authorization"):
 546                 cherrypy.response.headers[
 547                     "WWW-Authenticate"
 548                 ] = 'Bearer realm="Access to OSM site"'
 549                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 550         elif args and args[0] == "sleep":
 551             sleep_time = 5
 552
 553             try:
 554                 sleep_time = int(args[1])
 555             except Exception:
 556                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 557                 return self._format_out("Database already initialized")
 558
 559             thread_info = cherrypy.thread_data
 560             print(thread_info)
 561             time.sleep(sleep_time)
 562             # thread_info
 563         elif len(args) >= 2 and args[0] == "message":
 564             main_topic = args[1]
 565             return_text = "<html><pre>{} ->\n".format(main_topic)
 566
 567             try:
 568                 if cherrypy.request.method == "POST":
 569                     to_send = yaml.safe_load(cherrypy.request.body)
 570                     for k, v in to_send.items():
 571                         self.ns.msg.write(main_topic, k, v)
 572                         return_text += "  {}: {}\n".format(k, v)
 573                 elif cherrypy.request.method == "GET":
 574                     for k, v in kwargs.items():
 575                         self.ns.msg.write(main_topic, k, yaml.safe_load(v))
 576                         return_text += "  {}: {}\n".format(k, yaml.safe_load(v))
 577             except Exception as e:
 578                 return_text += "Error: " + str(e)
 579
 580             return_text += "</pre></html>\n"
 581
 582             return return_text
 583
 584         return_text = (
 585             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 586             + "  kwargs: {}\n".format(kwargs)
 587             + "  headers: {}\n".format(cherrypy.request.headers)
 588             + "  path_info: {}\n".format(cherrypy.request.path_info)
 589             + "  query_string: {}\n".format(cherrypy.request.query_string)
 590             + "  session: {}\n".format(cherrypy.session)
 591             + "  cookie: {}\n".format(cherrypy.request.cookie)
 592             + "  method: {}\n".format(cherrypy.request.method)
 593             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 594             + "  body:\n"
 595         )
 596         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 597
 598         if cherrypy.request.body.length:
 599             return_text += "    content: {}\n".format(
 600                 str(
 601                     cherrypy.request.body.read(
 602                         int(cherrypy.request.headers.get("Content-Length", 0))
 603                     )
 604                 )
 605             )
 606
 607         if thread_info:
 608             return_text += "thread: {}\n".format(thread_info)
 609
 610         return_text += "</pre></html>"
 611
 612         return return_text
 613
 614     @staticmethod
 615     def _check_valid_url_method(method, *args):
 616         if len(args) < 3:
 617             raise RoException(
 618                 "URL must contain at least 'main_topic/version/topic'",
 619                 HTTPStatus.METHOD_NOT_ALLOWED,
 620             )
 621
 622         reference = valid_url_methods
 623         for arg in args:
 624             if arg is None:
 625                 break
 626
 627             if not isinstance(reference, dict):
 628                 raise RoException(
 629                     "URL contains unexpected extra items '{}'".format(arg),
 630                     HTTPStatus.METHOD_NOT_ALLOWED,
 631                 )
 632
 633             if arg in reference:
 634                 reference = reference[arg]
 635             elif "<ID>" in reference:
 636                 reference = reference["<ID>"]
 637             elif "*" in reference:
 638                 # reference = reference["*"]
 639                 break
 640             else:
 641                 raise RoException(
 642                     "Unexpected URL item {}".format(arg),
 643                     HTTPStatus.METHOD_NOT_ALLOWED,
 644                 )
 645
 646         if "TODO" in reference and method in reference["TODO"]:
 647             raise RoException(
 648                 "Method {} not supported yet for this URL".format(method),
 649                 HTTPStatus.NOT_IMPLEMENTED,
 650             )
 651         elif "METHODS" not in reference or method not in reference["METHODS"]:
 652             raise RoException(
 653                 "Method {} not supported for this URL".format(method),
 654                 HTTPStatus.METHOD_NOT_ALLOWED,
 655             )
 656
 657         return reference["ROLE_PERMISSION"] + method.lower()
 658
 659     @staticmethod
 660     def _set_location_header(main_topic, version, topic, id):
 661         """
 662         Insert response header Location with the URL of created item base on URL params
 663         :param main_topic:
 664         :param version:
 665         :param topic:
 666         :param id:
 667         :return: None
 668         """
 669         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 670         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 671             main_topic, version, topic, id
 672         )
 673
 674         return
 675
 676     @cherrypy.expose
 677     def default(
 678         self,
 679         main_topic=None,
 680         version=None,
 681         topic=None,
 682         _id=None,
 683         _id2=None,
 684         *args,
 685         **kwargs,
 686     ):
 687         token_info = None
 688         outdata = None
 689         _format = None
 690         method = "DONE"
 691         rollback = []
 692         engine_session = None
 693
 694         try:
 695             if not main_topic or not version or not topic:
 696                 raise RoException(
 697                     "URL must contain at least 'main_topic/version/topic'",
 698                     HTTPStatus.METHOD_NOT_ALLOWED,
 699                 )
 700
 701             if main_topic not in (
 702                 "admin",
 703                 "ns",
 704             ):
 705                 raise RoException(
 706                     "URL main_topic '{}' not supported".format(main_topic),
 707                     HTTPStatus.METHOD_NOT_ALLOWED,
 708                 )
 709
 710             if version != "v1":
 711                 raise RoException(
 712                     "URL version '{}' not supported".format(version),
 713                     HTTPStatus.METHOD_NOT_ALLOWED,
 714                 )
 715
 716             if (
 717                 kwargs
 718                 and "METHOD" in kwargs
 719                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 720             ):
 721                 method = kwargs.pop("METHOD")
 722             else:
 723                 method = cherrypy.request.method
 724
 725             role_permission = self._check_valid_url_method(
 726                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 727             )
 728             # skip token validation if requesting a token
 729             indata = self._format_in(kwargs)
 730
 731             if main_topic != "admin" or topic != "tokens":
 732                 token_info = self.authenticator.authorize(role_permission, _id)
 733
 734             outdata, created_id, done = self.map_operation[role_permission](
 735                 engine_session, indata, version, _id, _id2, *args, *kwargs
 736             )
 737
 738             if created_id:
 739                 self._set_location_header(main_topic, version, topic, _id)
 740
 741             cherrypy.response.status = (
 742                 HTTPStatus.ACCEPTED.value
 743                 if not done
 744                 else HTTPStatus.OK.value
 745                 if outdata is not None
 746                 else HTTPStatus.NO_CONTENT.value
 747             )
 748
 749             return self._format_out(outdata, token_info, _format)
 750         except Exception as e:
 751             if isinstance(
 752                 e,
 753                 (
 754                     RoException,
 755                     NsException,
 756                     DbException,
 757                     FsException,
 758                     MsgException,
 759                     AuthException,
 760                     ValidationError,
 761                 ),
 762             ):
 763                 http_code_value = cherrypy.response.status = e.http_code.value
 764                 http_code_name = e.http_code.name
 765                 cherrypy.log("Exception {}".format(e))
 766             else:
 767                 http_code_value = (
 768                     cherrypy.response.status
 769                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 770                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 771                 http_code_name = HTTPStatus.BAD_REQUEST.name
 772
 773             if hasattr(outdata, "close"):  # is an open file
 774                 outdata.close()
 775
 776             error_text = str(e)
 777             rollback.reverse()
 778
 779             for rollback_item in rollback:
 780                 try:
 781                     if rollback_item.get("operation") == "set":
 782                         self.ns.db.set_one(
 783                             rollback_item["topic"],
 784                             {"_id": rollback_item["_id"]},
 785                             rollback_item["content"],
 786                             fail_on_empty=False,
 787                         )
 788                     else:
 789                         self.ns.db.del_one(
 790                             rollback_item["topic"],
 791                             {"_id": rollback_item["_id"]},
 792                             fail_on_empty=False,
 793                         )
 794                 except Exception as e2:
 795                     rollback_error_text = "Rollback Exception {}: {}".format(
 796                         rollback_item, e2
 797                     )
 798                     cherrypy.log(rollback_error_text)
 799                     error_text += ". " + rollback_error_text
 800
 801             # if isinstance(e, MsgException):
 802             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 803             #         engine_topic[:-1], method, error_text)
 804             problem_details = {
 805                 "code": http_code_name,
 806                 "status": http_code_value,
 807                 "detail": error_text,
 808             }
 809
 810             return self._format_out(problem_details, token_info)
 811             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 812         finally:
 813             if token_info:
 814                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 815                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 816                         if outdata.get(logging_id):
 817                             cherrypy.request.login += ";{}={}".format(
 818                                 logging_id, outdata[logging_id][:36]
 819                             )
 820
 821
 822 def _start_service():
 823     """
 824     Callback function called when cherrypy.engine starts
 825     Override configuration with env variables
 826     Set database, storage, message configuration
 827     Init database with admin/admin user password
 828     """
 829     global ro_server, vim_admin_thread
 830     # global vim_threads
 831     cherrypy.log.error("Starting osm_ng_ro")
 832     # update general cherrypy configuration
 833     update_dict = {}
 834     engine_config = cherrypy.tree.apps["/ro"].config
 835
 836     for k, v in environ.items():
 837         if not k.startswith("OSMRO_"):
 838             continue
 839
 840         k1, _, k2 = k[6:].lower().partition("_")
 841
 842         if not k2:
 843             continue
 844
 845         try:
 846             if k1 in ("server", "test", "auth", "log"):
 847                 # update [global] configuration
 848                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 849             elif k1 == "static":
 850                 # update [/static] configuration
 851                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 852             elif k1 == "tools":
 853                 # update [/] configuration
 854                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 855             elif k1 in ("message", "database", "storage", "authentication", "period"):
 856                 engine_config[k1][k2] = yaml.safe_load(v)
 857
 858         except Exception as e:
 859             raise RoException("Cannot load env '{}': {}".format(k, e))
 860
 861     if update_dict:
 862         cherrypy.config.update(update_dict)
 863         engine_config["global"].update(update_dict)
 864
 865     # logging cherrypy
 866     log_format_simple = (
 867         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 868     )
 869     log_formatter_simple = logging.Formatter(
 870         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 871     )
 872     logger_server = logging.getLogger("cherrypy.error")
 873     logger_access = logging.getLogger("cherrypy.access")
 874     logger_cherry = logging.getLogger("cherrypy")
 875     logger = logging.getLogger("ro")
 876
 877     if "log.file" in engine_config["global"]:
 878         file_handler = logging.handlers.RotatingFileHandler(
 879             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 880         )
 881         file_handler.setFormatter(log_formatter_simple)
 882         logger_cherry.addHandler(file_handler)
 883         logger.addHandler(file_handler)
 884
 885     # log always to standard output
 886     for format_, logger in {
 887         "ro.server %(filename)s:%(lineno)s": logger_server,
 888         "ro.access %(filename)s:%(lineno)s": logger_access,
 889         "%(name)s %(filename)s:%(lineno)s": logger,
 890     }.items():
 891         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 892         log_formatter_cherry = logging.Formatter(
 893             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 894         )
 895         str_handler = logging.StreamHandler()
 896         str_handler.setFormatter(log_formatter_cherry)
 897         logger.addHandler(str_handler)
 898
 899     if engine_config["global"].get("log.level"):
 900         logger_cherry.setLevel(engine_config["global"]["log.level"])
 901         logger.setLevel(engine_config["global"]["log.level"])
 902
 903     # logging other modules
 904     for k1, logname in {
 905         "message": "ro.msg",
 906         "database": "ro.db",
 907         "storage": "ro.fs",
 908     }.items():
 909         engine_config[k1]["logger_name"] = logname
 910         logger_module = logging.getLogger(logname)
 911
 912         if "logfile" in engine_config[k1]:
 913             file_handler = logging.handlers.RotatingFileHandler(
 914                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 915             )
 916             file_handler.setFormatter(log_formatter_simple)
 917             logger_module.addHandler(file_handler)
 918
 919         if "loglevel" in engine_config[k1]:
 920             logger_module.setLevel(engine_config[k1]["loglevel"])
 921     # TODO add more entries, e.g.: storage
 922
 923     engine_config["assignment"] = {}
 924     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 925     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 926     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 927     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 928
 929     # # start subscriptions thread:
 930     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 931     vim_admin_thread.start()
 932     # # Do not capture except SubscriptionException
 933
 934     # backend = engine_config["authentication"]["backend"]
 935     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 936     #                    .format(ro_version, ro_version_date, backend))
 937
 938
 939 def _stop_service():
 940     """
 941     Callback function called when cherrypy.engine stops
 942     TODO: Ending database connections.
 943     """
 944     global vim_admin_thread
 945
 946     # terminate vim_admin_thread
 947     if vim_admin_thread:
 948         vim_admin_thread.terminate()
 949
 950     vim_admin_thread = None
 951     cherrypy.tree.apps["/ro"].root.ns.stop()
 952     cherrypy.log.error("Stopping osm_ng_ro")
 953
 954
 955 def ro_main(config_file):
 956     global ro_server
 957
 958     ro_server = Server()
 959     cherrypy.engine.subscribe("start", _start_service)
 960     cherrypy.engine.subscribe("stop", _stop_service)
 961     cherrypy.quickstart(ro_server, "/ro", config_file)
 962
 963
 964 def usage():
 965     print(
 966         """Usage: {} [options]
 967         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 968         -h|--help: shows this help
 969         """.format(
 970             sys.argv[0]
 971         )
 972     )
 973     # --log-socket-host HOST: send logs to this host")
 974     # --log-socket-port PORT: send logs using this port (default: 9022)")
 975
 976
 977 if __name__ == "__main__":
 978     try:
 979         # load parameters and configuration
 980         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 981         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 982         config_file = None
 983
 984         for o, a in opts:
 985             if o in ("-h", "--help"):
 986                 usage()
 987                 sys.exit()
 988             elif o in ("-c", "--config"):
 989                 config_file = a
 990             else:
 991                 raise ValueError("Unhandled option")
 992
 993         if config_file:
 994             if not path.isfile(config_file):
 995                 print(
 996                     "configuration file '{}' that not exist".format(config_file),
 997                     file=sys.stderr,
 998                 )
 999                 exit(1)
1000         else:
1001             for config_file in (
1002                 path.dirname(__file__) + "/ro.cfg",
1003                 "./ro.cfg",
1004                 "/etc/osm/ro.cfg",
1005             ):
1006                 if path.isfile(config_file):
1007                     break
1008             else:
1009                 print(
1010                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1011                     file=sys.stderr,
1012                 )
1013                 exit(1)
1014
1015         ro_main(config_file)
1016     except KeyboardInterrupt:
1017         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1018     except getopt.GetoptError as e:
1019         print(str(e), file=sys.stderr)
1020         # usage()
1021         exit(1)