NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "rebuild": {
  87                 "METHODS": ("POST",),
  88                 "ROLE_PERMISSION": "rebuild:",
  89                 "<ID>": {
  90                     "METHODS": ("POST",),
  91                     "ROLE_PERMISSION": "rebuild:id:",
  92                 },
  93             },
  94             "start": {
  95                 "METHODS": ("POST",),
  96                 "ROLE_PERMISSION": "start:",
  97                 "<ID>": {
  98                     "METHODS": ("POST",),
  99                     "ROLE_PERMISSION": "start:id:",
 100                 },
 101             },
 102             "stop": {
 103                 "METHODS": ("POST",),
 104                 "ROLE_PERMISSION": "stop:",
 105                 "<ID>": {
 106                     "METHODS": ("POST",),
 107                     "ROLE_PERMISSION": "stop:id:",
 108                 },
 109             },
 110             "deploy": {
 111                 "METHODS": ("GET",),
 112                 "ROLE_PERMISSION": "deploy:",
 113                 "<ID>": {
 114                     "METHODS": ("GET", "POST", "DELETE"),
 115                     "ROLE_PERMISSION": "deploy:id:",
 116                     "<ID>": {
 117                         "METHODS": ("GET",),
 118                         "ROLE_PERMISSION": "deploy:id:id:",
 119                         "cancel": {
 120                             "METHODS": ("POST",),
 121                             "ROLE_PERMISSION": "deploy:id:id:cancel",
 122                         },
 123                     },
 124                 },
 125             },
 126             "recreate": {
 127                 "<ID>": {
 128                     "METHODS": ("POST"),
 129                     "ROLE_PERMISSION": "recreate:id:",
 130                     "<ID>": {
 131                         "METHODS": ("GET",),
 132                         "ROLE_PERMISSION": "recreate:id:id:",
 133                     },
 134                 },
 135             },
 136             "migrate": {
 137                 "<ID>": {
 138                     "METHODS": ("POST"),
 139                     "ROLE_PERMISSION": "migrate:id:",
 140                     "<ID>": {
 141                         "METHODS": ("GET",),
 142                         "ROLE_PERMISSION": "migrate:id:id:",
 143                     },
 144                 },
 145             },
 146         }
 147     },
 148 }
 149
 150
 151 class RoException(Exception):
 152     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 153         Exception.__init__(self, message)
 154         self.http_code = http_code
 155
 156
 157 class AuthException(RoException):
 158     pass
 159
 160
 161 class Authenticator:
 162     def __init__(self, valid_url_methods, valid_query_string):
 163         self.valid_url_methods = valid_url_methods
 164         self.valid_query_string = valid_query_string
 165
 166     def authorize(self, *args, **kwargs):
 167         return {"token": "ok", "id": "ok"}
 168
 169     def new_token(self, token_info, indata, remote):
 170         return {"token": "ok", "id": "ok", "remote": remote}
 171
 172     def del_token(self, token_id):
 173         pass
 174
 175     def start(self, engine_config):
 176         pass
 177
 178
 179 class Server(object):
 180     instance = 0
 181     # to decode bytes to str
 182     reader = getreader("utf-8")
 183
 184     def __init__(self):
 185         self.instance += 1
 186         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 187         self.ns = Ns()
 188         self.map_operation = {
 189             "token:post": self.new_token,
 190             "token:id:delete": self.del_token,
 191             "deploy:get": self.ns.get_deploy,
 192             "deploy:id:get": self.ns.get_actions,
 193             "deploy:id:post": self.ns.deploy,
 194             "deploy:id:delete": self.ns.delete,
 195             "deploy:id:id:get": self.ns.status,
 196             "deploy:id:id:cancel:post": self.ns.cancel,
 197             "rebuild:id:post": self.ns.rebuild_start_stop,
 198             "start:id:post": self.ns.rebuild_start_stop,
 199             "stop:id:post": self.ns.rebuild_start_stop,
 200             "recreate:id:post": self.ns.recreate,
 201             "recreate:id:id:get": self.ns.recreate_status,
 202             "migrate:id:post": self.ns.migrate,
 203         }
 204
 205     def _format_in(self, kwargs):
 206         try:
 207             indata = None
 208
 209             if cherrypy.request.body.length:
 210                 error_text = "Invalid input format "
 211
 212                 if "Content-Type" in cherrypy.request.headers:
 213                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 214                         error_text = "Invalid json format "
 215                         indata = json.load(self.reader(cherrypy.request.body))
 216                         cherrypy.request.headers.pop("Content-File-MD5", None)
 217                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 218                         error_text = "Invalid yaml format "
 219                         indata = yaml.load(
 220                             cherrypy.request.body, Loader=yaml.SafeLoader
 221                         )
 222                         cherrypy.request.headers.pop("Content-File-MD5", None)
 223                     elif (
 224                         "application/binary" in cherrypy.request.headers["Content-Type"]
 225                         or "application/gzip"
 226                         in cherrypy.request.headers["Content-Type"]
 227                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 228                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 229                     ):
 230                         indata = cherrypy.request.body  # .read()
 231                     elif (
 232                         "multipart/form-data"
 233                         in cherrypy.request.headers["Content-Type"]
 234                     ):
 235                         if "descriptor_file" in kwargs:
 236                             filecontent = kwargs.pop("descriptor_file")
 237
 238                             if not filecontent.file:
 239                                 raise RoException(
 240                                     "empty file or content", HTTPStatus.BAD_REQUEST
 241                                 )
 242
 243                             indata = filecontent.file  # .read()
 244
 245                             if filecontent.content_type.value:
 246                                 cherrypy.request.headers[
 247                                     "Content-Type"
 248                                 ] = filecontent.content_type.value
 249                     else:
 250                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 251                         #                          "Only 'Content-Type' of type 'application/json' or
 252                         # 'application/yaml' for input format are available")
 253                         error_text = "Invalid yaml format "
 254                         indata = yaml.load(
 255                             cherrypy.request.body, Loader=yaml.SafeLoader
 256                         )
 257                         cherrypy.request.headers.pop("Content-File-MD5", None)
 258                 else:
 259                     error_text = "Invalid yaml format "
 260                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 261                     cherrypy.request.headers.pop("Content-File-MD5", None)
 262
 263             if not indata:
 264                 indata = {}
 265
 266             format_yaml = False
 267             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 268                 format_yaml = True
 269
 270             for k, v in kwargs.items():
 271                 if isinstance(v, str):
 272                     if v == "":
 273                         kwargs[k] = None
 274                     elif format_yaml:
 275                         try:
 276                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 277                         except Exception:
 278                             pass
 279                     elif (
 280                         k.endswith(".gt")
 281                         or k.endswith(".lt")
 282                         or k.endswith(".gte")
 283                         or k.endswith(".lte")
 284                     ):
 285                         try:
 286                             kwargs[k] = int(v)
 287                         except Exception:
 288                             try:
 289                                 kwargs[k] = float(v)
 290                             except Exception:
 291                                 pass
 292                     elif v.find(",") > 0:
 293                         kwargs[k] = v.split(",")
 294                 elif isinstance(v, (list, tuple)):
 295                     for index in range(0, len(v)):
 296                         if v[index] == "":
 297                             v[index] = None
 298                         elif format_yaml:
 299                             try:
 300                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 301                             except Exception:
 302                                 pass
 303
 304             return indata
 305         except (ValueError, yaml.YAMLError) as exc:
 306             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 307         except KeyError as exc:
 308             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 309         except Exception as exc:
 310             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 311
 312     @staticmethod
 313     def _format_out(data, token_info=None, _format=None):
 314         """
 315         return string of dictionary data according to requested json, yaml, xml. By default json
 316         :param data: response to be sent. Can be a dict, text or file
 317         :param token_info: Contains among other username and project
 318         :param _format: The format to be set as Content-Type if data is a file
 319         :return: None
 320         """
 321         accept = cherrypy.request.headers.get("Accept")
 322
 323         if data is None:
 324             if accept and "text/html" in accept:
 325                 return html.format(
 326                     data, cherrypy.request, cherrypy.response, token_info
 327                 )
 328
 329             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 330             return
 331         elif hasattr(data, "read"):  # file object
 332             if _format:
 333                 cherrypy.response.headers["Content-Type"] = _format
 334             elif "b" in data.mode:  # binariy asssumig zip
 335                 cherrypy.response.headers["Content-Type"] = "application/zip"
 336             else:
 337                 cherrypy.response.headers["Content-Type"] = "text/plain"
 338
 339             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 340             return data
 341
 342         if accept:
 343             if "application/json" in accept:
 344                 cherrypy.response.headers[
 345                     "Content-Type"
 346                 ] = "application/json; charset=utf-8"
 347                 a = json.dumps(data, indent=4) + "\n"
 348
 349                 return a.encode("utf8")
 350             elif "text/html" in accept:
 351                 return html.format(
 352                     data, cherrypy.request, cherrypy.response, token_info
 353                 )
 354             elif (
 355                 "application/yaml" in accept
 356                 or "*/*" in accept
 357                 or "text/plain" in accept
 358             ):
 359                 pass
 360             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 361             elif cherrypy.response.status >= 400:
 362                 raise cherrypy.HTTPError(
 363                     HTTPStatus.NOT_ACCEPTABLE.value,
 364                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 365                     "for output format are available",
 366                 )
 367
 368         cherrypy.response.headers["Content-Type"] = "application/yaml"
 369
 370         return yaml.safe_dump(
 371             data,
 372             explicit_start=True,
 373             indent=4,
 374             default_flow_style=False,
 375             tags=False,
 376             encoding="utf-8",
 377             allow_unicode=True,
 378         )  # , canonical=True, default_style='"'
 379
 380     @cherrypy.expose
 381     def index(self, *args, **kwargs):
 382         token_info = None
 383
 384         try:
 385             if cherrypy.request.method == "GET":
 386                 token_info = self.authenticator.authorize()
 387                 outdata = token_info  # Home page
 388             else:
 389                 raise cherrypy.HTTPError(
 390                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 391                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 392                 )
 393
 394             return self._format_out(outdata, token_info)
 395         except (NsException, AuthException) as e:
 396             # cherrypy.log("index Exception {}".format(e))
 397             cherrypy.response.status = e.http_code.value
 398
 399             return self._format_out("Welcome to OSM!", token_info)
 400
 401     @cherrypy.expose
 402     def version(self, *args, **kwargs):
 403         # TODO consider to remove and provide version using the static version file
 404         try:
 405             if cherrypy.request.method != "GET":
 406                 raise RoException(
 407                     "Only method GET is allowed",
 408                     HTTPStatus.METHOD_NOT_ALLOWED,
 409                 )
 410             elif args or kwargs:
 411                 raise RoException(
 412                     "Invalid URL or query string for version",
 413                     HTTPStatus.METHOD_NOT_ALLOWED,
 414                 )
 415
 416             # TODO include version of other modules, pick up from some kafka admin message
 417             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 418
 419             return self._format_out(osm_ng_ro_version)
 420         except RoException as e:
 421             cherrypy.response.status = e.http_code.value
 422             problem_details = {
 423                 "code": e.http_code.name,
 424                 "status": e.http_code.value,
 425                 "detail": str(e),
 426             }
 427
 428             return self._format_out(problem_details, None)
 429
 430     def new_token(self, engine_session, indata, *args, **kwargs):
 431         token_info = None
 432
 433         try:
 434             token_info = self.authenticator.authorize()
 435         except Exception:
 436             token_info = None
 437
 438         if kwargs:
 439             indata.update(kwargs)
 440
 441         # This is needed to log the user when authentication fails
 442         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 443         token_info = self.authenticator.new_token(
 444             token_info, indata, cherrypy.request.remote
 445         )
 446         cherrypy.session["Authorization"] = token_info["id"]
 447         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 448         # for logging
 449
 450         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 451         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 452
 453         return token_info, token_info["id"], True
 454
 455     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 456         token_id = _id
 457
 458         if not token_id and "id" in kwargs:
 459             token_id = kwargs["id"]
 460         elif not token_id:
 461             token_info = self.authenticator.authorize()
 462             # for logging
 463             token_id = token_info["id"]
 464
 465         self.authenticator.del_token(token_id)
 466         token_info = None
 467         cherrypy.session["Authorization"] = "logout"
 468         # cherrypy.response.cookie["Authorization"] = token_id
 469         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 470
 471         return None, None, True
 472
 473     @cherrypy.expose
 474     def test(self, *args, **kwargs):
 475         if not cherrypy.config.get("server.enable_test") or (
 476             isinstance(cherrypy.config["server.enable_test"], str)
 477             and cherrypy.config["server.enable_test"].lower() == "false"
 478         ):
 479             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 480
 481             return "test URL is disabled"
 482
 483         thread_info = None
 484
 485         if args and args[0] == "help":
 486             return (
 487                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 488                 "sleep/<time>\nmessage/topic\n</pre></html>"
 489             )
 490         elif args and args[0] == "init":
 491             try:
 492                 # self.ns.load_dbase(cherrypy.request.app.config)
 493                 self.ns.create_admin()
 494
 495                 return "Done. User 'admin', password 'admin' created"
 496             except Exception:
 497                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 498
 499                 return self._format_out("Database already initialized")
 500         elif args and args[0] == "file":
 501             return cherrypy.lib.static.serve_file(
 502                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 503                 "text/plain",
 504                 "attachment",
 505             )
 506         elif args and args[0] == "file2":
 507             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 508             f = open(f_path, "r")
 509             cherrypy.response.headers["Content-type"] = "text/plain"
 510             return f
 511
 512         elif len(args) == 2 and args[0] == "db-clear":
 513             deleted_info = self.ns.db.del_list(args[1], kwargs)
 514             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 515         elif len(args) and args[0] == "fs-clear":
 516             if len(args) >= 2:
 517                 folders = (args[1],)
 518             else:
 519                 folders = self.ns.fs.dir_ls(".")
 520
 521             for folder in folders:
 522                 self.ns.fs.file_delete(folder)
 523
 524             return ",".join(folders) + " folders deleted\n"
 525         elif args and args[0] == "login":
 526             if not cherrypy.request.headers.get("Authorization"):
 527                 cherrypy.response.headers[
 528                     "WWW-Authenticate"
 529                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 530                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 531         elif args and args[0] == "login2":
 532             if not cherrypy.request.headers.get("Authorization"):
 533                 cherrypy.response.headers[
 534                     "WWW-Authenticate"
 535                 ] = 'Bearer realm="Access to OSM site"'
 536                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 537         elif args and args[0] == "sleep":
 538             sleep_time = 5
 539
 540             try:
 541                 sleep_time = int(args[1])
 542             except Exception:
 543                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 544                 return self._format_out("Database already initialized")
 545
 546             thread_info = cherrypy.thread_data
 547             print(thread_info)
 548             time.sleep(sleep_time)
 549             # thread_info
 550         elif len(args) >= 2 and args[0] == "message":
 551             main_topic = args[1]
 552             return_text = "<html><pre>{} ->\n".format(main_topic)
 553
 554             try:
 555                 if cherrypy.request.method == "POST":
 556                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 557                     for k, v in to_send.items():
 558                         self.ns.msg.write(main_topic, k, v)
 559                         return_text += "  {}: {}\n".format(k, v)
 560                 elif cherrypy.request.method == "GET":
 561                     for k, v in kwargs.items():
 562                         self.ns.msg.write(
 563                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 564                         )
 565                         return_text += "  {}: {}\n".format(
 566                             k, yaml.load(v, Loader=yaml.SafeLoader)
 567                         )
 568             except Exception as e:
 569                 return_text += "Error: " + str(e)
 570
 571             return_text += "</pre></html>\n"
 572
 573             return return_text
 574
 575         return_text = (
 576             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 577             + "  kwargs: {}\n".format(kwargs)
 578             + "  headers: {}\n".format(cherrypy.request.headers)
 579             + "  path_info: {}\n".format(cherrypy.request.path_info)
 580             + "  query_string: {}\n".format(cherrypy.request.query_string)
 581             + "  session: {}\n".format(cherrypy.session)
 582             + "  cookie: {}\n".format(cherrypy.request.cookie)
 583             + "  method: {}\n".format(cherrypy.request.method)
 584             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 585             + "  body:\n"
 586         )
 587         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 588
 589         if cherrypy.request.body.length:
 590             return_text += "    content: {}\n".format(
 591                 str(
 592                     cherrypy.request.body.read(
 593                         int(cherrypy.request.headers.get("Content-Length", 0))
 594                     )
 595                 )
 596             )
 597
 598         if thread_info:
 599             return_text += "thread: {}\n".format(thread_info)
 600
 601         return_text += "</pre></html>"
 602
 603         return return_text
 604
 605     @staticmethod
 606     def _check_valid_url_method(method, *args):
 607         if len(args) < 3:
 608             raise RoException(
 609                 "URL must contain at least 'main_topic/version/topic'",
 610                 HTTPStatus.METHOD_NOT_ALLOWED,
 611             )
 612
 613         reference = valid_url_methods
 614         for arg in args:
 615             if arg is None:
 616                 break
 617
 618             if not isinstance(reference, dict):
 619                 raise RoException(
 620                     "URL contains unexpected extra items '{}'".format(arg),
 621                     HTTPStatus.METHOD_NOT_ALLOWED,
 622                 )
 623
 624             if arg in reference:
 625                 reference = reference[arg]
 626             elif "<ID>" in reference:
 627                 reference = reference["<ID>"]
 628             elif "*" in reference:
 629                 # reference = reference["*"]
 630                 break
 631             else:
 632                 raise RoException(
 633                     "Unexpected URL item {}".format(arg),
 634                     HTTPStatus.METHOD_NOT_ALLOWED,
 635                 )
 636
 637         if "TODO" in reference and method in reference["TODO"]:
 638             raise RoException(
 639                 "Method {} not supported yet for this URL".format(method),
 640                 HTTPStatus.NOT_IMPLEMENTED,
 641             )
 642         elif "METHODS" not in reference or method not in reference["METHODS"]:
 643             raise RoException(
 644                 "Method {} not supported for this URL".format(method),
 645                 HTTPStatus.METHOD_NOT_ALLOWED,
 646             )
 647
 648         return reference["ROLE_PERMISSION"] + method.lower()
 649
 650     @staticmethod
 651     def _set_location_header(main_topic, version, topic, id):
 652         """
 653         Insert response header Location with the URL of created item base on URL params
 654         :param main_topic:
 655         :param version:
 656         :param topic:
 657         :param id:
 658         :return: None
 659         """
 660         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 661         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 662             main_topic, version, topic, id
 663         )
 664
 665         return
 666
 667     @cherrypy.expose
 668     def default(
 669         self,
 670         main_topic=None,
 671         version=None,
 672         topic=None,
 673         _id=None,
 674         _id2=None,
 675         *args,
 676         **kwargs,
 677     ):
 678         token_info = None
 679         outdata = None
 680         _format = None
 681         method = "DONE"
 682         rollback = []
 683         engine_session = None
 684
 685         try:
 686             if not main_topic or not version or not topic:
 687                 raise RoException(
 688                     "URL must contain at least 'main_topic/version/topic'",
 689                     HTTPStatus.METHOD_NOT_ALLOWED,
 690                 )
 691
 692             if main_topic not in (
 693                 "admin",
 694                 "ns",
 695             ):
 696                 raise RoException(
 697                     "URL main_topic '{}' not supported".format(main_topic),
 698                     HTTPStatus.METHOD_NOT_ALLOWED,
 699                 )
 700
 701             if version != "v1":
 702                 raise RoException(
 703                     "URL version '{}' not supported".format(version),
 704                     HTTPStatus.METHOD_NOT_ALLOWED,
 705                 )
 706
 707             if (
 708                 kwargs
 709                 and "METHOD" in kwargs
 710                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 711             ):
 712                 method = kwargs.pop("METHOD")
 713             else:
 714                 method = cherrypy.request.method
 715
 716             role_permission = self._check_valid_url_method(
 717                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 718             )
 719             # skip token validation if requesting a token
 720             indata = self._format_in(kwargs)
 721
 722             if main_topic != "admin" or topic != "tokens":
 723                 token_info = self.authenticator.authorize(role_permission, _id)
 724
 725             outdata, created_id, done = self.map_operation[role_permission](
 726                 engine_session, indata, version, _id, _id2, *args, *kwargs
 727             )
 728
 729             if created_id:
 730                 self._set_location_header(main_topic, version, topic, _id)
 731
 732             cherrypy.response.status = (
 733                 HTTPStatus.ACCEPTED.value
 734                 if not done
 735                 else HTTPStatus.OK.value
 736                 if outdata is not None
 737                 else HTTPStatus.NO_CONTENT.value
 738             )
 739
 740             return self._format_out(outdata, token_info, _format)
 741         except Exception as e:
 742             if isinstance(
 743                 e,
 744                 (
 745                     RoException,
 746                     NsException,
 747                     DbException,
 748                     FsException,
 749                     MsgException,
 750                     AuthException,
 751                     ValidationError,
 752                 ),
 753             ):
 754                 http_code_value = cherrypy.response.status = e.http_code.value
 755                 http_code_name = e.http_code.name
 756                 cherrypy.log("Exception {}".format(e))
 757             else:
 758                 http_code_value = (
 759                     cherrypy.response.status
 760                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 761                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 762                 http_code_name = HTTPStatus.BAD_REQUEST.name
 763
 764             if hasattr(outdata, "close"):  # is an open file
 765                 outdata.close()
 766
 767             error_text = str(e)
 768             rollback.reverse()
 769
 770             for rollback_item in rollback:
 771                 try:
 772                     if rollback_item.get("operation") == "set":
 773                         self.ns.db.set_one(
 774                             rollback_item["topic"],
 775                             {"_id": rollback_item["_id"]},
 776                             rollback_item["content"],
 777                             fail_on_empty=False,
 778                         )
 779                     else:
 780                         self.ns.db.del_one(
 781                             rollback_item["topic"],
 782                             {"_id": rollback_item["_id"]},
 783                             fail_on_empty=False,
 784                         )
 785                 except Exception as e2:
 786                     rollback_error_text = "Rollback Exception {}: {}".format(
 787                         rollback_item, e2
 788                     )
 789                     cherrypy.log(rollback_error_text)
 790                     error_text += ". " + rollback_error_text
 791
 792             # if isinstance(e, MsgException):
 793             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 794             #         engine_topic[:-1], method, error_text)
 795             problem_details = {
 796                 "code": http_code_name,
 797                 "status": http_code_value,
 798                 "detail": error_text,
 799             }
 800
 801             return self._format_out(problem_details, token_info)
 802             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 803         finally:
 804             if token_info:
 805                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 806                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 807                         if outdata.get(logging_id):
 808                             cherrypy.request.login += ";{}={}".format(
 809                                 logging_id, outdata[logging_id][:36]
 810                             )
 811
 812
 813 def _start_service():
 814     """
 815     Callback function called when cherrypy.engine starts
 816     Override configuration with env variables
 817     Set database, storage, message configuration
 818     Init database with admin/admin user password
 819     """
 820     global ro_server, vim_admin_thread
 821     # global vim_threads
 822     cherrypy.log.error("Starting osm_ng_ro")
 823     # update general cherrypy configuration
 824     update_dict = {}
 825     engine_config = cherrypy.tree.apps["/ro"].config
 826
 827     for k, v in environ.items():
 828         if not k.startswith("OSMRO_"):
 829             continue
 830
 831         k1, _, k2 = k[6:].lower().partition("_")
 832
 833         if not k2:
 834             continue
 835
 836         try:
 837             if k1 in ("server", "test", "auth", "log"):
 838                 # update [global] configuration
 839                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 840             elif k1 == "static":
 841                 # update [/static] configuration
 842                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 843             elif k1 == "tools":
 844                 # update [/] configuration
 845                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 846             elif k1 in ("message", "database", "storage", "authentication"):
 847                 engine_config[k1][k2] = yaml.safe_load(v)
 848
 849         except Exception as e:
 850             raise RoException("Cannot load env '{}': {}".format(k, e))
 851
 852     if update_dict:
 853         cherrypy.config.update(update_dict)
 854         engine_config["global"].update(update_dict)
 855
 856     # logging cherrypy
 857     log_format_simple = (
 858         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 859     )
 860     log_formatter_simple = logging.Formatter(
 861         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 862     )
 863     logger_server = logging.getLogger("cherrypy.error")
 864     logger_access = logging.getLogger("cherrypy.access")
 865     logger_cherry = logging.getLogger("cherrypy")
 866     logger = logging.getLogger("ro")
 867
 868     if "log.file" in engine_config["global"]:
 869         file_handler = logging.handlers.RotatingFileHandler(
 870             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 871         )
 872         file_handler.setFormatter(log_formatter_simple)
 873         logger_cherry.addHandler(file_handler)
 874         logger.addHandler(file_handler)
 875
 876     # log always to standard output
 877     for format_, logger in {
 878         "ro.server %(filename)s:%(lineno)s": logger_server,
 879         "ro.access %(filename)s:%(lineno)s": logger_access,
 880         "%(name)s %(filename)s:%(lineno)s": logger,
 881     }.items():
 882         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 883         log_formatter_cherry = logging.Formatter(
 884             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 885         )
 886         str_handler = logging.StreamHandler()
 887         str_handler.setFormatter(log_formatter_cherry)
 888         logger.addHandler(str_handler)
 889
 890     if engine_config["global"].get("log.level"):
 891         logger_cherry.setLevel(engine_config["global"]["log.level"])
 892         logger.setLevel(engine_config["global"]["log.level"])
 893
 894     # logging other modules
 895     for k1, logname in {
 896         "message": "ro.msg",
 897         "database": "ro.db",
 898         "storage": "ro.fs",
 899     }.items():
 900         engine_config[k1]["logger_name"] = logname
 901         logger_module = logging.getLogger(logname)
 902
 903         if "logfile" in engine_config[k1]:
 904             file_handler = logging.handlers.RotatingFileHandler(
 905                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 906             )
 907             file_handler.setFormatter(log_formatter_simple)
 908             logger_module.addHandler(file_handler)
 909
 910         if "loglevel" in engine_config[k1]:
 911             logger_module.setLevel(engine_config[k1]["loglevel"])
 912     # TODO add more entries, e.g.: storage
 913
 914     engine_config["assignment"] = {}
 915     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 916     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 917     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 918     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 919
 920     # # start subscriptions thread:
 921     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 922     vim_admin_thread.start()
 923     # # Do not capture except SubscriptionException
 924
 925     # backend = engine_config["authentication"]["backend"]
 926     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 927     #                    .format(ro_version, ro_version_date, backend))
 928
 929
 930 def _stop_service():
 931     """
 932     Callback function called when cherrypy.engine stops
 933     TODO: Ending database connections.
 934     """
 935     global vim_admin_thread
 936
 937     # terminate vim_admin_thread
 938     if vim_admin_thread:
 939         vim_admin_thread.terminate()
 940
 941     vim_admin_thread = None
 942     cherrypy.tree.apps["/ro"].root.ns.stop()
 943     cherrypy.log.error("Stopping osm_ng_ro")
 944
 945
 946 def ro_main(config_file):
 947     global ro_server
 948
 949     ro_server = Server()
 950     cherrypy.engine.subscribe("start", _start_service)
 951     cherrypy.engine.subscribe("stop", _stop_service)
 952     cherrypy.quickstart(ro_server, "/ro", config_file)
 953
 954
 955 def usage():
 956     print(
 957         """Usage: {} [options]
 958         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 959         -h|--help: shows this help
 960         """.format(
 961             sys.argv[0]
 962         )
 963     )
 964     # --log-socket-host HOST: send logs to this host")
 965     # --log-socket-port PORT: send logs using this port (default: 9022)")
 966
 967
 968 if __name__ == "__main__":
 969     try:
 970         # load parameters and configuration
 971         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 972         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 973         config_file = None
 974
 975         for o, a in opts:
 976             if o in ("-h", "--help"):
 977                 usage()
 978                 sys.exit()
 979             elif o in ("-c", "--config"):
 980                 config_file = a
 981             else:
 982                 assert False, "Unhandled option"
 983
 984         if config_file:
 985             if not path.isfile(config_file):
 986                 print(
 987                     "configuration file '{}' that not exist".format(config_file),
 988                     file=sys.stderr,
 989                 )
 990                 exit(1)
 991         else:
 992             for config_file in (
 993                 path.dirname(__file__) + "/ro.cfg",
 994                 "./ro.cfg",
 995                 "/etc/osm/ro.cfg",
 996             ):
 997                 if path.isfile(config_file):
 998                     break
 999             else:
1000                 print(
1001                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1002                     file=sys.stderr,
1003                 )
1004                 exit(1)
1005
1006         ro_main(config_file)
1007     except KeyboardInterrupt:
1008         print("KeyboardInterrupt. Finishing", file=sys.stderr)
1009     except getopt.GetoptError as e:
1010         print(str(e), file=sys.stderr)
1011         # usage()
1012         exit(1)