NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "rebuild": {
  87                 "METHODS": ("POST",),
  88                 "ROLE_PERMISSION": "rebuild:",
  89                 "<ID>": {
  90                     "METHODS": ("POST",),
  91                     "ROLE_PERMISSION": "rebuild:id:",
  92                 },
  93             },
  94             "start": {
  95                 "METHODS": ("POST",),
  96                 "ROLE_PERMISSION": "start:",
  97                 "<ID>": {
  98                     "METHODS": ("POST",),
  99                     "ROLE_PERMISSION": "start:id:",
 100                 },
 101             },
 102             "stop": {
 103                 "METHODS": ("POST",),
 104                 "ROLE_PERMISSION": "stop:",
 105                 "<ID>": {
 106                     "METHODS": ("POST",),
 107                     "ROLE_PERMISSION": "stop:id:",
 108                 },
 109             },
 110             "deploy": {
 111                 "METHODS": ("GET",),
 112                 "ROLE_PERMISSION": "deploy:",
 113                 "<ID>": {
 114                     "METHODS": ("GET", "POST", "DELETE"),
 115                     "ROLE_PERMISSION": "deploy:id:",
 116                     "<ID>": {
 117                         "METHODS": ("GET",),
 118                         "ROLE_PERMISSION": "deploy:id:id:",
 119                         "cancel": {
 120                             "METHODS": ("POST",),
 121                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 122                         },
 123                     },
 124                 },
 125             },
 126             "recreate": {
 127                 "<ID>": {
 128                     "METHODS": ("POST"),
 129                     "ROLE_PERMISSION": "recreate:id:",
 130                     "<ID>": {
 131                         "METHODS": ("GET",),
 132                         "ROLE_PERMISSION": "recreate:id:id:",
 133                     },
 134                 },
 135             },
 136             "migrate": {
 137                 "<ID>": {
 138                     "METHODS": ("POST"),
 139                     "ROLE_PERMISSION": "migrate:id:",
 140                     "<ID>": {
 141                         "METHODS": ("GET",),
 142                         "ROLE_PERMISSION": "migrate:id:id:",
 143                     },
 144                 },
 145             },
 146             "verticalscale": {
 147                 "<ID>": {
 148                     "METHODS": ("POST"),
 149                     "ROLE_PERMISSION": "verticalscale:id:",
 150                     "<ID>": {
 151                         "METHODS": ("GET",),
 152                         "ROLE_PERMISSION": "verticalscale:id:id:",
 153                     },
 154                 },
 155             },
 156         }
 157     },
 158 }
 159
 160
 161 class RoException(Exception):
 162     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 163         Exception.__init__(self, message)
 164         self.http_code = http_code
 165
 166
 167 class AuthException(RoException):
 168     pass
 169
 170
 171 class Authenticator:
 172     def __init__(self, valid_url_methods, valid_query_string):
 173         self.valid_url_methods = valid_url_methods
 174         self.valid_query_string = valid_query_string
 175
 176     def authorize(self, *args, **kwargs):
 177         return {"token": "ok", "id": "ok"}
 178
 179     def new_token(self, token_info, indata, remote):
 180         return {"token": "ok", "id": "ok", "remote": remote}
 181
 182     def del_token(self, token_id):
 183         pass
 184
 185     def start(self, engine_config):
 186         pass
 187
 188
 189 class Server(object):
 190     instance = 0
 191     # to decode bytes to str
 192     reader = getreader("utf-8")
 193
 194     def __init__(self):
 195         self.instance += 1
 196         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 197         self.ns = Ns()
 198         self.map_operation = {
 199             "token:post": self.new_token,
 200             "token:id:delete": self.del_token,
 201             "deploy:get": self.ns.get_deploy,
 202             "deploy:id:get": self.ns.get_actions,
 203             "deploy:id:post": self.ns.deploy,
 204             "deploy:id:delete": self.ns.delete,
 205             "deploy:id:id:get": self.ns.status,
 206             "deploy:id:id:cancel:post": self.ns.cancel,
 207             "rebuild:id:post": self.ns.rebuild_start_stop,
 208             "start:id:post": self.ns.rebuild_start_stop,
 209             "stop:id:post": self.ns.rebuild_start_stop,
 210             "recreate:id:post": self.ns.recreate,
 211             "recreate:id:id:get": self.ns.recreate_status,
 212             "migrate:id:post": self.ns.migrate,
 213             "verticalscale:id:post": self.ns.verticalscale,
 214         }
 215
 216     def _format_in(self, kwargs):
 217         try:
 218             indata = None
 219
 220             if cherrypy.request.body.length:
 221                 error_text = "Invalid input format "
 222
 223                 if "Content-Type" in cherrypy.request.headers:
 224                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 225                         error_text = "Invalid json format "
 226                         indata = json.load(self.reader(cherrypy.request.body))
 227                         cherrypy.request.headers.pop("Content-File-MD5", None)
 228                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 229                         error_text = "Invalid yaml format "
 230                         indata = yaml.load(
 231                             cherrypy.request.body, Loader=yaml.SafeLoader
 232                         )
 233                         cherrypy.request.headers.pop("Content-File-MD5", None)
 234                     elif (
 235                         "application/binary" in cherrypy.request.headers["Content-Type"]
 236                         or "application/gzip"
 237                         in cherrypy.request.headers["Content-Type"]
 238                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 239                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 240                     ):
 241                         indata = cherrypy.request.body  # .read()
 242                     elif (
 243                         "multipart/form-data"
 244                         in cherrypy.request.headers["Content-Type"]
 245                     ):
 246                         if "descriptor_file" in kwargs:
 247                             filecontent = kwargs.pop("descriptor_file")
 248
 249                             if not filecontent.file:
 250                                 raise RoException(
 251                                     "empty file or content", HTTPStatus.BAD_REQUEST
 252                                 )
 253
 254                             indata = filecontent.file  # .read()
 255
 256                             if filecontent.content_type.value:
 257                                 cherrypy.request.headers[
 258                                     "Content-Type"
 259                                 ] = filecontent.content_type.value
 260                     else:
 261                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 262                         #                          "Only 'Content-Type' of type 'application/json' or
 263                         # 'application/yaml' for input format are available")
 264                         error_text = "Invalid yaml format "
 265                         indata = yaml.load(
 266                             cherrypy.request.body, Loader=yaml.SafeLoader
 267                         )
 268                         cherrypy.request.headers.pop("Content-File-MD5", None)
 269                 else:
 270                     error_text = "Invalid yaml format "
 271                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 272                     cherrypy.request.headers.pop("Content-File-MD5", None)
 273
 274             if not indata:
 275                 indata = {}
 276
 277             format_yaml = False
 278             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 279                 format_yaml = True
 280
 281             for k, v in kwargs.items():
 282                 if isinstance(v, str):
 283                     if v == "":
 284                         kwargs[k] = None
 285                     elif format_yaml:
 286                         try:
 287                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 288                         except Exception:
 289                             pass
 290                     elif (
 291                         k.endswith(".gt")
 292                         or k.endswith(".lt")
 293                         or k.endswith(".gte")
 294                         or k.endswith(".lte")
 295                     ):
 296                         try:
 297                             kwargs[k] = int(v)
 298                         except Exception:
 299                             try:
 300                                 kwargs[k] = float(v)
 301                             except Exception:
 302                                 pass
 303                     elif v.find(",") > 0:
 304                         kwargs[k] = v.split(",")
 305                 elif isinstance(v, (list, tuple)):
 306                     for index in range(0, len(v)):
 307                         if v[index] == "":
 308                             v[index] = None
 309                         elif format_yaml:
 310                             try:
 311                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 312                             except Exception:
 313                                 pass
 314
 315             return indata
 316         except (ValueError, yaml.YAMLError) as exc:
 317             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 318         except KeyError as exc:
 319             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 320         except Exception as exc:
 321             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 322
 323     @staticmethod
 324     def _format_out(data, token_info=None, _format=None):
 325         """
 326         return string of dictionary data according to requested json, yaml, xml. By default json
 327         :param data: response to be sent. Can be a dict, text or file
 328         :param token_info: Contains among other username and project
 329         :param _format: The format to be set as Content-Type if data is a file
 330         :return: None
 331         """
 332         accept = cherrypy.request.headers.get("Accept")
 333
 334         if data is None:
 335             if accept and "text/html" in accept:
 336                 return html.format(
 337                     data, cherrypy.request, cherrypy.response, token_info
 338                 )
 339
 340             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 341             return
 342         elif hasattr(data, "read"):  # file object
 343             if _format:
 344                 cherrypy.response.headers["Content-Type"] = _format
 345             elif "b" in data.mode:  # binariy asssumig zip
 346                 cherrypy.response.headers["Content-Type"] = "application/zip"
 347             else:
 348                 cherrypy.response.headers["Content-Type"] = "text/plain"
 349
 350             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 351             return data
 352
 353         if accept:
 354             if "application/json" in accept:
 355                 cherrypy.response.headers[
 356                     "Content-Type"
 357                 ] = "application/json; charset=utf-8"
 358                 a = json.dumps(data, indent=4) + "\n"
 359
 360                 return a.encode("utf8")
 361             elif "text/html" in accept:
 362                 return html.format(
 363                     data, cherrypy.request, cherrypy.response, token_info
 364                 )
 365             elif (
 366                 "application/yaml" in accept
 367                 or "*/*" in accept
 368                 or "text/plain" in accept
 369             ):
 370                 pass
 371             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 372             elif cherrypy.response.status >= 400:
 373                 raise cherrypy.HTTPError(
 374                     HTTPStatus.NOT_ACCEPTABLE.value,
 375                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 376                     "for output format are available",
 377                 )
 378
 379         cherrypy.response.headers["Content-Type"] = "application/yaml"
 380
 381         return yaml.safe_dump(
 382             data,
 383             explicit_start=True,
 384             indent=4,
 385             default_flow_style=False,
 386             tags=False,
 387             encoding="utf-8",
 388             allow_unicode=True,
 389         )  # , canonical=True, default_style='"'
 390
 391     @cherrypy.expose
 392     def index(self, *args, **kwargs):
 393         token_info = None
 394
 395         try:
 396             if cherrypy.request.method == "GET":
 397                 token_info = self.authenticator.authorize()
 398                 outdata = token_info  # Home page
 399             else:
 400                 raise cherrypy.HTTPError(
 401                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 402                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 403                 )
 404
 405             return self._format_out(outdata, token_info)
 406         except (NsException, AuthException) as e:
 407             # cherrypy.log("index Exception {}".format(e))
 408             cherrypy.response.status = e.http_code.value
 409
 410             return self._format_out("Welcome to OSM!", token_info)
 411
 412     @cherrypy.expose
 413     def version(self, *args, **kwargs):
 414         # TODO consider to remove and provide version using the static version file
 415         try:
 416             if cherrypy.request.method != "GET":
 417                 raise RoException(
 418                     "Only method GET is allowed",
 419                     HTTPStatus.METHOD_NOT_ALLOWED,
 420                 )
 421             elif args or kwargs:
 422                 raise RoException(
 423                     "Invalid URL or query string for version",
 424                     HTTPStatus.METHOD_NOT_ALLOWED,
 425                 )
 426
 427             # TODO include version of other modules, pick up from some kafka admin message
 428             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 429
 430             return self._format_out(osm_ng_ro_version)
 431         except RoException as e:
 432             cherrypy.response.status = e.http_code.value
 433             problem_details = {
 434                 "code": e.http_code.name,
 435                 "status": e.http_code.value,
 436                 "detail": str(e),
 437             }
 438
 439             return self._format_out(problem_details, None)
 440
 441     def new_token(self, engine_session, indata, *args, **kwargs):
 442         token_info = None
 443
 444         try:
 445             token_info = self.authenticator.authorize()
 446         except Exception:
 447             token_info = None
 448
 449         if kwargs:
 450             indata.update(kwargs)
 451
 452         # This is needed to log the user when authentication fails
 453         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 454         token_info = self.authenticator.new_token(
 455             token_info, indata, cherrypy.request.remote
 456         )
 457         cherrypy.session["Authorization"] = token_info["id"]
 458         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 459         # for logging
 460
 461         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 462         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 463
 464         return token_info, token_info["id"], True
 465
 466     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 467         token_id = _id
 468
 469         if not token_id and "id" in kwargs:
 470             token_id = kwargs["id"]
 471         elif not token_id:
 472             token_info = self.authenticator.authorize()
 473             # for logging
 474             token_id = token_info["id"]
 475
 476         self.authenticator.del_token(token_id)
 477         token_info = None
 478         cherrypy.session["Authorization"] = "logout"
 479         # cherrypy.response.cookie["Authorization"] = token_id
 480         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 481
 482         return None, None, True
 483
 484     @cherrypy.expose
 485     def test(self, *args, **kwargs):
 486         if not cherrypy.config.get("server.enable_test") or (
 487             isinstance(cherrypy.config["server.enable_test"], str)
 488             and cherrypy.config["server.enable_test"].lower() == "false"
 489         ):
 490             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 491
 492             return "test URL is disabled"
 493
 494         thread_info = None
 495
 496         if args and args[0] == "help":
 497             return (
 498                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 499                 "sleep/<time>\nmessage/topic\n</pre></html>"
 500             )
 501         elif args and args[0] == "init":
 502             try:
 503                 # self.ns.load_dbase(cherrypy.request.app.config)
 504                 self.ns.create_admin()
 505
 506                 return "Done. User 'admin', password 'admin' created"
 507             except Exception:
 508                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 509
 510                 return self._format_out("Database already initialized")
 511         elif args and args[0] == "file":
 512             return cherrypy.lib.static.serve_file(
 513                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 514                 "text/plain",
 515                 "attachment",
 516             )
 517         elif args and args[0] == "file2":
 518             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 519             f = open(f_path, "r")
 520             cherrypy.response.headers["Content-type"] = "text/plain"
 521             return f
 522
 523         elif len(args) == 2 and args[0] == "db-clear":
 524             deleted_info = self.ns.db.del_list(args[1], kwargs)
 525             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 526         elif len(args) and args[0] == "fs-clear":
 527             if len(args) >= 2:
 528                 folders = (args[1],)
 529             else:
 530                 folders = self.ns.fs.dir_ls(".")
 531
 532             for folder in folders:
 533                 self.ns.fs.file_delete(folder)
 534
 535             return ",".join(folders) + " folders deleted\n"
 536         elif args and args[0] == "login":
 537             if not cherrypy.request.headers.get("Authorization"):
 538                 cherrypy.response.headers[
 539                     "WWW-Authenticate"
 540                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 541                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 542         elif args and args[0] == "login2":
 543             if not cherrypy.request.headers.get("Authorization"):
 544                 cherrypy.response.headers[
 545                     "WWW-Authenticate"
 546                 ] = 'Bearer realm="Access to OSM site"'
 547                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 548         elif args and args[0] == "sleep":
 549             sleep_time = 5
 550
 551             try:
 552                 sleep_time = int(args[1])
 553             except Exception:
 554                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 555                 return self._format_out("Database already initialized")
 556
 557             thread_info = cherrypy.thread_data
 558             print(thread_info)
 559             time.sleep(sleep_time)
 560             # thread_info
 561         elif len(args) >= 2 and args[0] == "message":
 562             main_topic = args[1]
 563             return_text = "<html><pre>{} ->\n".format(main_topic)
 564
 565             try:
 566                 if cherrypy.request.method == "POST":
 567                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 568                     for k, v in to_send.items():
 569                         self.ns.msg.write(main_topic, k, v)
 570                         return_text += "  {}: {}\n".format(k, v)
 571                 elif cherrypy.request.method == "GET":
 572                     for k, v in kwargs.items():
 573                         self.ns.msg.write(
 574                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 575                         )
 576                         return_text += "  {}: {}\n".format(
 577                             k, yaml.load(v, Loader=yaml.SafeLoader)
 578                         )
 579             except Exception as e:
 580                 return_text += "Error: " + str(e)
 581
 582             return_text += "</pre></html>\n"
 583
 584             return return_text
 585
 586         return_text = (
 587             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 588             + "  kwargs: {}\n".format(kwargs)
 589             + "  headers: {}\n".format(cherrypy.request.headers)
 590             + "  path_info: {}\n".format(cherrypy.request.path_info)
 591             + "  query_string: {}\n".format(cherrypy.request.query_string)
 592             + "  session: {}\n".format(cherrypy.session)
 593             + "  cookie: {}\n".format(cherrypy.request.cookie)
 594             + "  method: {}\n".format(cherrypy.request.method)
 595             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 596             + "  body:\n"
 597         )
 598         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 599
 600         if cherrypy.request.body.length:
 601             return_text += "    content: {}\n".format(
 602                 str(
 603                     cherrypy.request.body.read(
 604                         int(cherrypy.request.headers.get("Content-Length", 0))
 605                     )
 606                 )
 607             )
 608
 609         if thread_info:
 610             return_text += "thread: {}\n".format(thread_info)
 611
 612         return_text += "</pre></html>"
 613
 614         return return_text
 615
 616     @staticmethod
 617     def _check_valid_url_method(method, *args):
 618         if len(args) < 3:
 619             raise RoException(
 620                 "URL must contain at least 'main_topic/version/topic'",
 621                 HTTPStatus.METHOD_NOT_ALLOWED,
 622             )
 623
 624         reference = valid_url_methods
 625         for arg in args:
 626             if arg is None:
 627                 break
 628
 629             if not isinstance(reference, dict):
 630                 raise RoException(
 631                     "URL contains unexpected extra items '{}'".format(arg),
 632                     HTTPStatus.METHOD_NOT_ALLOWED,
 633                 )
 634
 635             if arg in reference:
 636                 reference = reference[arg]
 637             elif "<ID>" in reference:
 638                 reference = reference["<ID>"]
 639             elif "*" in reference:
 640                 # reference = reference["*"]
 641                 break
 642             else:
 643                 raise RoException(
 644                     "Unexpected URL item {}".format(arg),
 645                     HTTPStatus.METHOD_NOT_ALLOWED,
 646                 )
 647
 648         if "TODO" in reference and method in reference["TODO"]:
 649             raise RoException(
 650                 "Method {} not supported yet for this URL".format(method),
 651                 HTTPStatus.NOT_IMPLEMENTED,
 652             )
 653         elif "METHODS" not in reference or method not in reference["METHODS"]:
 654             raise RoException(
 655                 "Method {} not supported for this URL".format(method),
 656                 HTTPStatus.METHOD_NOT_ALLOWED,
 657             )
 658
 659         return reference["ROLE_PERMISSION"] + method.lower()
 660
 661     @staticmethod
 662     def _set_location_header(main_topic, version, topic, id):
 663         """
 664         Insert response header Location with the URL of created item base on URL params
 665         :param main_topic:
 666         :param version:
 667         :param topic:
 668         :param id:
 669         :return: None
 670         """
 671         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 672         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 673             main_topic, version, topic, id
 674         )
 675
 676         return
 677
 678     @cherrypy.expose
 679     def default(
 680         self,
 681         main_topic=None,
 682         version=None,
 683         topic=None,
 684         _id=None,
 685         _id2=None,
 686         *args,
 687         **kwargs,
 688     ):
 689         token_info = None
 690         outdata = None
 691         _format = None
 692         method = "DONE"
 693         rollback = []
 694         engine_session = None
 695
 696         try:
 697             if not main_topic or not version or not topic:
 698                 raise RoException(
 699                     "URL must contain at least 'main_topic/version/topic'",
 700                     HTTPStatus.METHOD_NOT_ALLOWED,
 701                 )
 702
 703             if main_topic not in (
 704                 "admin",
 705                 "ns",
 706             ):
 707                 raise RoException(
 708                     "URL main_topic '{}' not supported".format(main_topic),
 709                     HTTPStatus.METHOD_NOT_ALLOWED,
 710                 )
 711
 712             if version != "v1":
 713                 raise RoException(
 714                     "URL version '{}' not supported".format(version),
 715                     HTTPStatus.METHOD_NOT_ALLOWED,
 716                 )
 717
 718             if (
 719                 kwargs
 720                 and "METHOD" in kwargs
 721                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 722             ):
 723                 method = kwargs.pop("METHOD")
 724             else:
 725                 method = cherrypy.request.method
 726
 727             role_permission = self._check_valid_url_method(
 728                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 729             )
 730             # skip token validation if requesting a token
 731             indata = self._format_in(kwargs)
 732
 733             if main_topic != "admin" or topic != "tokens":
 734                 token_info = self.authenticator.authorize(role_permission, _id)
 735
 736             outdata, created_id, done = self.map_operation[role_permission](
 737                 engine_session, indata, version, _id, _id2, *args, *kwargs
 738             )
 739
 740             if created_id:
 741                 self._set_location_header(main_topic, version, topic, _id)
 742
 743             cherrypy.response.status = (
 744                 HTTPStatus.ACCEPTED.value
 745                 if not done
 746                 else HTTPStatus.OK.value
 747                 if outdata is not None
 748                 else HTTPStatus.NO_CONTENT.value
 749             )
 750
 751             return self._format_out(outdata, token_info, _format)
 752         except Exception as e:
 753             if isinstance(
 754                 e,
 755                 (
 756                     RoException,
 757                     NsException,
 758                     DbException,
 759                     FsException,
 760                     MsgException,
 761                     AuthException,
 762                     ValidationError,
 763                 ),
 764             ):
 765                 http_code_value = cherrypy.response.status = e.http_code.value
 766                 http_code_name = e.http_code.name
 767                 cherrypy.log("Exception {}".format(e))
 768             else:
 769                 http_code_value = (
 770                     cherrypy.response.status
 771                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 772                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 773                 http_code_name = HTTPStatus.BAD_REQUEST.name
 774
 775             if hasattr(outdata, "close"):  # is an open file
 776                 outdata.close()
 777
 778             error_text = str(e)
 779             rollback.reverse()
 780
 781             for rollback_item in rollback:
 782                 try:
 783                     if rollback_item.get("operation") == "set":
 784                         self.ns.db.set_one(
 785                             rollback_item["topic"],
 786                             {"_id": rollback_item["_id"]},
 787                             rollback_item["content"],
 788                             fail_on_empty=False,
 789                         )
 790                     else:
 791                         self.ns.db.del_one(
 792                             rollback_item["topic"],
 793                             {"_id": rollback_item["_id"]},
 794                             fail_on_empty=False,
 795                         )
 796                 except Exception as e2:
 797                     rollback_error_text = "Rollback Exception {}: {}".format(
 798                         rollback_item, e2
 799                     )
 800                     cherrypy.log(rollback_error_text)
 801                     error_text += ". " + rollback_error_text
 802
 803             # if isinstance(e, MsgException):
 804             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 805             #         engine_topic[:-1], method, error_text)
 806             problem_details = {
 807                 "code": http_code_name,
 808                 "status": http_code_value,
 809                 "detail": error_text,
 810             }
 811
 812             return self._format_out(problem_details, token_info)
 813             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 814         finally:
 815             if token_info:
 816                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 817                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 818                         if outdata.get(logging_id):
 819                             cherrypy.request.login += ";{}={}".format(
 820                                 logging_id, outdata[logging_id][:36]
 821                             )
 822
 823
 824 def _start_service():
 825     """
 826     Callback function called when cherrypy.engine starts
 827     Override configuration with env variables
 828     Set database, storage, message configuration
 829     Init database with admin/admin user password
 830     """
 831     global ro_server, vim_admin_thread
 832     # global vim_threads
 833     cherrypy.log.error("Starting osm_ng_ro")
 834     # update general cherrypy configuration
 835     update_dict = {}
 836     engine_config = cherrypy.tree.apps["/ro"].config
 837
 838     for k, v in environ.items():
 839         if not k.startswith("OSMRO_"):
 840             continue
 841
 842         k1, _, k2 = k[6:].lower().partition("_")
 843
 844         if not k2:
 845             continue
 846
 847         try:
 848             if k1 in ("server", "test", "auth", "log"):
 849                 # update [global] configuration
 850                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 851             elif k1 == "static":
 852                 # update [/static] configuration
 853                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 854             elif k1 == "tools":
 855                 # update [/] configuration
 856                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 857             elif k1 in ("message", "database", "storage", "authentication"):
 858                 engine_config[k1][k2] = yaml.safe_load(v)
 859
 860         except Exception as e:
 861             raise RoException("Cannot load env '{}': {}".format(k, e))
 862
 863     if update_dict:
 864         cherrypy.config.update(update_dict)
 865         engine_config["global"].update(update_dict)
 866
 867     # logging cherrypy
 868     log_format_simple = (
 869         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 870     )
 871     log_formatter_simple = logging.Formatter(
 872         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 873     )
 874     logger_server = logging.getLogger("cherrypy.error")
 875     logger_access = logging.getLogger("cherrypy.access")
 876     logger_cherry = logging.getLogger("cherrypy")
 877     logger = logging.getLogger("ro")
 878
 879     if "log.file" in engine_config["global"]:
 880         file_handler = logging.handlers.RotatingFileHandler(
 881             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 882         )
 883         file_handler.setFormatter(log_formatter_simple)
 884         logger_cherry.addHandler(file_handler)
 885         logger.addHandler(file_handler)
 886
 887     # log always to standard output
 888     for format_, logger in {
 889         "ro.server %(filename)s:%(lineno)s": logger_server,
 890         "ro.access %(filename)s:%(lineno)s": logger_access,
 891         "%(name)s %(filename)s:%(lineno)s": logger,
 892     }.items():
 893         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 894         log_formatter_cherry = logging.Formatter(
 895             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 896         )
 897         str_handler = logging.StreamHandler()
 898         str_handler.setFormatter(log_formatter_cherry)
 899         logger.addHandler(str_handler)
 900
 901     if engine_config["global"].get("log.level"):
 902         logger_cherry.setLevel(engine_config["global"]["log.level"])
 903         logger.setLevel(engine_config["global"]["log.level"])
 904
 905     # logging other modules
 906     for k1, logname in {
 907         "message": "ro.msg",
 908         "database": "ro.db",
 909         "storage": "ro.fs",
 910     }.items():
 911         engine_config[k1]["logger_name"] = logname
 912         logger_module = logging.getLogger(logname)
 913
 914         if "logfile" in engine_config[k1]:
 915             file_handler = logging.handlers.RotatingFileHandler(
 916                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 917             )
 918             file_handler.setFormatter(log_formatter_simple)
 919             logger_module.addHandler(file_handler)
 920
 921         if "loglevel" in engine_config[k1]:
 922             logger_module.setLevel(engine_config[k1]["loglevel"])
 923     # TODO add more entries, e.g.: storage
 924
 925     engine_config["assignment"] = {}
 926     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 927     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 928     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 929     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 930
 931     # # start subscriptions thread:
 932     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 933     vim_admin_thread.start()
 934     # # Do not capture except SubscriptionException
 935
 936     # backend = engine_config["authentication"]["backend"]
 937     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 938     #                    .format(ro_version, ro_version_date, backend))
 939
 940
 941 def _stop_service():
 942     """
 943     Callback function called when cherrypy.engine stops
 944     TODO: Ending database connections.
 945     """
 946     global vim_admin_thread
 947
 948     # terminate vim_admin_thread
 949     if vim_admin_thread:
 950         vim_admin_thread.terminate()
 951
 952     vim_admin_thread = None
 953     cherrypy.tree.apps["/ro"].root.ns.stop()
 954     cherrypy.log.error("Stopping osm_ng_ro")
 955
 956
 957 def ro_main(config_file):
 958     global ro_server
 959
 960     ro_server = Server()
 961     cherrypy.engine.subscribe("start", _start_service)
 962     cherrypy.engine.subscribe("stop", _stop_service)
 963     cherrypy.quickstart(ro_server, "/ro", config_file)
 964
 965
 966 def usage():
 967     print(
 968         """Usage: {} [options]
 969         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 970         -h|--help: shows this help
 971         """.format(
 972             sys.argv[0]
 973         )
 974     )
 975     # --log-socket-host HOST: send logs to this host")
 976     # --log-socket-port PORT: send logs using this port (default: 9022)")
 977
 978
 979 if __name__ == "__main__":
 980     try:
 981         # load parameters and configuration
 982         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 983         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 984         config_file = None
 985
 986         for o, a in opts:
 987             if o in ("-h", "--help"):
 988                 usage()
 989                 sys.exit()
 990             elif o in ("-c", "--config"):
 991                 config_file = a
 992             else:
 993                 assert False, "Unhandled option"
 994
 995         if config_file:
 996             if not path.isfile(config_file):
 997                 print(
 998                     "configuration file '{}' that not exist".format(config_file),
 999                     file=sys.stderr,
1000                 )
1001                 exit(1)
1002         else:
1003             for config_file in (
1004                 path.dirname(__file__) + "/ro.cfg",
1005                 "./ro.cfg",
1006                 "/etc/osm/ro.cfg",
1007             ):
1008                 if path.isfile(config_file):
1009                     break
1010             else:
1011                 print(
1012                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1013                     file=sys.stderr,
1014                 )
1015                 exit(1)
1016
1017         ro_main(config_file)
1018     except KeyboardInterrupt:
1019         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1020     except getopt.GetoptError as e:
1021         print(str(e), file=sys.stderr)
1022         # usage()
1023         exit(1)