NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "deploy": {
  87                 "METHODS": ("GET",),
  88                 "ROLE_PERMISSION": "deploy:",
  89                 "<ID>": {
  90                     "METHODS": ("GET", "POST", "DELETE"),
  91                     "ROLE_PERMISSION": "deploy:id:",
  92                     "<ID>": {
  93                         "METHODS": ("GET",),
  94                         "ROLE_PERMISSION": "deploy:id:id:",
  95                         "cancel": {
  96                             "METHODS": ("POST",),
  97                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  98                         },
  99                     },
 100                 },
 101             },
 102         }
 103     },
 104 }
 105
 106
 107 class RoException(Exception):
 108     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 109         Exception.__init__(self, message)
 110         self.http_code = http_code
 111
 112
 113 class AuthException(RoException):
 114     pass
 115
 116
 117 class Authenticator:
 118     def __init__(self, valid_url_methods, valid_query_string):
 119         self.valid_url_methods = valid_url_methods
 120         self.valid_query_string = valid_query_string
 121
 122     def authorize(self, *args, **kwargs):
 123         return {"token": "ok", "id": "ok"}
 124
 125     def new_token(self, token_info, indata, remote):
 126         return {"token": "ok", "id": "ok", "remote": remote}
 127
 128     def del_token(self, token_id):
 129         pass
 130
 131     def start(self, engine_config):
 132         pass
 133
 134
 135 class Server(object):
 136     instance = 0
 137     # to decode bytes to str
 138     reader = getreader("utf-8")
 139
 140     def __init__(self):
 141         self.instance += 1
 142         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 143         self.ns = Ns()
 144         self.map_operation = {
 145             "token:post": self.new_token,
 146             "token:id:delete": self.del_token,
 147             "deploy:get": self.ns.get_deploy,
 148             "deploy:id:get": self.ns.get_actions,
 149             "deploy:id:post": self.ns.deploy,
 150             "deploy:id:delete": self.ns.delete,
 151             "deploy:id:id:get": self.ns.status,
 152             "deploy:id:id:cancel:post": self.ns.cancel,
 153         }
 154
 155     def _format_in(self, kwargs):
 156         try:
 157             indata = None
 158
 159             if cherrypy.request.body.length:
 160                 error_text = "Invalid input format "
 161
 162                 if "Content-Type" in cherrypy.request.headers:
 163                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 164                         error_text = "Invalid json format "
 165                         indata = json.load(self.reader(cherrypy.request.body))
 166                         cherrypy.request.headers.pop("Content-File-MD5", None)
 167                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 168                         error_text = "Invalid yaml format "
 169                         indata = yaml.load(
 170                             cherrypy.request.body, Loader=yaml.SafeLoader
 171                         )
 172                         cherrypy.request.headers.pop("Content-File-MD5", None)
 173                     elif (
 174                         "application/binary" in cherrypy.request.headers["Content-Type"]
 175                         or "application/gzip"
 176                         in cherrypy.request.headers["Content-Type"]
 177                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 178                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 179                     ):
 180                         indata = cherrypy.request.body  # .read()
 181                     elif (
 182                         "multipart/form-data"
 183                         in cherrypy.request.headers["Content-Type"]
 184                     ):
 185                         if "descriptor_file" in kwargs:
 186                             filecontent = kwargs.pop("descriptor_file")
 187
 188                             if not filecontent.file:
 189                                 raise RoException(
 190                                     "empty file or content", HTTPStatus.BAD_REQUEST
 191                                 )
 192
 193                             indata = filecontent.file  # .read()
 194
 195                             if filecontent.content_type.value:
 196                                 cherrypy.request.headers[
 197                                     "Content-Type"
 198                                 ] = filecontent.content_type.value
 199                     else:
 200                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 201                         #                          "Only 'Content-Type' of type 'application/json' or
 202                         # 'application/yaml' for input format are available")
 203                         error_text = "Invalid yaml format "
 204                         indata = yaml.load(
 205                             cherrypy.request.body, Loader=yaml.SafeLoader
 206                         )
 207                         cherrypy.request.headers.pop("Content-File-MD5", None)
 208                 else:
 209                     error_text = "Invalid yaml format "
 210                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 211                     cherrypy.request.headers.pop("Content-File-MD5", None)
 212
 213             if not indata:
 214                 indata = {}
 215
 216             format_yaml = False
 217             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 218                 format_yaml = True
 219
 220             for k, v in kwargs.items():
 221                 if isinstance(v, str):
 222                     if v == "":
 223                         kwargs[k] = None
 224                     elif format_yaml:
 225                         try:
 226                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 227                         except Exception:
 228                             pass
 229                     elif (
 230                         k.endswith(".gt")
 231                         or k.endswith(".lt")
 232                         or k.endswith(".gte")
 233                         or k.endswith(".lte")
 234                     ):
 235                         try:
 236                             kwargs[k] = int(v)
 237                         except Exception:
 238                             try:
 239                                 kwargs[k] = float(v)
 240                             except Exception:
 241                                 pass
 242                     elif v.find(",") > 0:
 243                         kwargs[k] = v.split(",")
 244                 elif isinstance(v, (list, tuple)):
 245                     for index in range(0, len(v)):
 246                         if v[index] == "":
 247                             v[index] = None
 248                         elif format_yaml:
 249                             try:
 250                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 251                             except Exception:
 252                                 pass
 253
 254             return indata
 255         except (ValueError, yaml.YAMLError) as exc:
 256             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 257         except KeyError as exc:
 258             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 259         except Exception as exc:
 260             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 261
 262     @staticmethod
 263     def _format_out(data, token_info=None, _format=None):
 264         """
 265         return string of dictionary data according to requested json, yaml, xml. By default json
 266         :param data: response to be sent. Can be a dict, text or file
 267         :param token_info: Contains among other username and project
 268         :param _format: The format to be set as Content-Type if data is a file
 269         :return: None
 270         """
 271         accept = cherrypy.request.headers.get("Accept")
 272
 273         if data is None:
 274             if accept and "text/html" in accept:
 275                 return html.format(
 276                     data, cherrypy.request, cherrypy.response, token_info
 277                 )
 278
 279             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 280             return
 281         elif hasattr(data, "read"):  # file object
 282             if _format:
 283                 cherrypy.response.headers["Content-Type"] = _format
 284             elif "b" in data.mode:  # binariy asssumig zip
 285                 cherrypy.response.headers["Content-Type"] = "application/zip"
 286             else:
 287                 cherrypy.response.headers["Content-Type"] = "text/plain"
 288
 289             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 290             return data
 291
 292         if accept:
 293             if "application/json" in accept:
 294                 cherrypy.response.headers[
 295                     "Content-Type"
 296                 ] = "application/json; charset=utf-8"
 297                 a = json.dumps(data, indent=4) + "\n"
 298
 299                 return a.encode("utf8")
 300             elif "text/html" in accept:
 301                 return html.format(
 302                     data, cherrypy.request, cherrypy.response, token_info
 303                 )
 304             elif (
 305                 "application/yaml" in accept
 306                 or "*/*" in accept
 307                 or "text/plain" in accept
 308             ):
 309                 pass
 310             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 311             elif cherrypy.response.status >= 400:
 312                 raise cherrypy.HTTPError(
 313                     HTTPStatus.NOT_ACCEPTABLE.value,
 314                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 315                     "for output format are available",
 316                 )
 317
 318         cherrypy.response.headers["Content-Type"] = "application/yaml"
 319
 320         return yaml.safe_dump(
 321             data,
 322             explicit_start=True,
 323             indent=4,
 324             default_flow_style=False,
 325             tags=False,
 326             encoding="utf-8",
 327             allow_unicode=True,
 328         )  # , canonical=True, default_style='"'
 329
 330     @cherrypy.expose
 331     def index(self, *args, **kwargs):
 332         token_info = None
 333
 334         try:
 335             if cherrypy.request.method == "GET":
 336                 token_info = self.authenticator.authorize()
 337                 outdata = token_info  # Home page
 338             else:
 339                 raise cherrypy.HTTPError(
 340                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 341                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 342                 )
 343
 344             return self._format_out(outdata, token_info)
 345         except (NsException, AuthException) as e:
 346             # cherrypy.log("index Exception {}".format(e))
 347             cherrypy.response.status = e.http_code.value
 348
 349             return self._format_out("Welcome to OSM!", token_info)
 350
 351     @cherrypy.expose
 352     def version(self, *args, **kwargs):
 353         # TODO consider to remove and provide version using the static version file
 354         try:
 355             if cherrypy.request.method != "GET":
 356                 raise RoException(
 357                     "Only method GET is allowed",
 358                     HTTPStatus.METHOD_NOT_ALLOWED,
 359                 )
 360             elif args or kwargs:
 361                 raise RoException(
 362                     "Invalid URL or query string for version",
 363                     HTTPStatus.METHOD_NOT_ALLOWED,
 364                 )
 365
 366             # TODO include version of other modules, pick up from some kafka admin message
 367             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 368
 369             return self._format_out(osm_ng_ro_version)
 370         except RoException as e:
 371             cherrypy.response.status = e.http_code.value
 372             problem_details = {
 373                 "code": e.http_code.name,
 374                 "status": e.http_code.value,
 375                 "detail": str(e),
 376             }
 377
 378             return self._format_out(problem_details, None)
 379
 380     def new_token(self, engine_session, indata, *args, **kwargs):
 381         token_info = None
 382
 383         try:
 384             token_info = self.authenticator.authorize()
 385         except Exception:
 386             token_info = None
 387
 388         if kwargs:
 389             indata.update(kwargs)
 390
 391         # This is needed to log the user when authentication fails
 392         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 393         token_info = self.authenticator.new_token(
 394             token_info, indata, cherrypy.request.remote
 395         )
 396         cherrypy.session["Authorization"] = token_info["id"]
 397         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 398         # for logging
 399
 400         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 401         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 402
 403         return token_info, token_info["id"], True
 404
 405     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 406         token_id = _id
 407
 408         if not token_id and "id" in kwargs:
 409             token_id = kwargs["id"]
 410         elif not token_id:
 411             token_info = self.authenticator.authorize()
 412             # for logging
 413             token_id = token_info["id"]
 414
 415         self.authenticator.del_token(token_id)
 416         token_info = None
 417         cherrypy.session["Authorization"] = "logout"
 418         # cherrypy.response.cookie["Authorization"] = token_id
 419         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 420
 421         return None, None, True
 422
 423     @cherrypy.expose
 424     def test(self, *args, **kwargs):
 425         if not cherrypy.config.get("server.enable_test") or (
 426             isinstance(cherrypy.config["server.enable_test"], str)
 427             and cherrypy.config["server.enable_test"].lower() == "false"
 428         ):
 429             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 430
 431             return "test URL is disabled"
 432
 433         thread_info = None
 434
 435         if args and args[0] == "help":
 436             return (
 437                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 438                 "sleep/<time>\nmessage/topic\n</pre></html>"
 439             )
 440         elif args and args[0] == "init":
 441             try:
 442                 # self.ns.load_dbase(cherrypy.request.app.config)
 443                 self.ns.create_admin()
 444
 445                 return "Done. User 'admin', password 'admin' created"
 446             except Exception:
 447                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 448
 449                 return self._format_out("Database already initialized")
 450         elif args and args[0] == "file":
 451             return cherrypy.lib.static.serve_file(
 452                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 453                 "text/plain",
 454                 "attachment",
 455             )
 456         elif args and args[0] == "file2":
 457             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 458             f = open(f_path, "r")
 459             cherrypy.response.headers["Content-type"] = "text/plain"
 460             return f
 461
 462         elif len(args) == 2 and args[0] == "db-clear":
 463             deleted_info = self.ns.db.del_list(args[1], kwargs)
 464             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 465         elif len(args) and args[0] == "fs-clear":
 466             if len(args) >= 2:
 467                 folders = (args[1],)
 468             else:
 469                 folders = self.ns.fs.dir_ls(".")
 470
 471             for folder in folders:
 472                 self.ns.fs.file_delete(folder)
 473
 474             return ",".join(folders) + " folders deleted\n"
 475         elif args and args[0] == "login":
 476             if not cherrypy.request.headers.get("Authorization"):
 477                 cherrypy.response.headers[
 478                     "WWW-Authenticate"
 479                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 480                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 481         elif args and args[0] == "login2":
 482             if not cherrypy.request.headers.get("Authorization"):
 483                 cherrypy.response.headers[
 484                     "WWW-Authenticate"
 485                 ] = 'Bearer realm="Access to OSM site"'
 486                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 487         elif args and args[0] == "sleep":
 488             sleep_time = 5
 489
 490             try:
 491                 sleep_time = int(args[1])
 492             except Exception:
 493                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 494                 return self._format_out("Database already initialized")
 495
 496             thread_info = cherrypy.thread_data
 497             print(thread_info)
 498             time.sleep(sleep_time)
 499             # thread_info
 500         elif len(args) >= 2 and args[0] == "message":
 501             main_topic = args[1]
 502             return_text = "<html><pre>{} ->\n".format(main_topic)
 503
 504             try:
 505                 if cherrypy.request.method == "POST":
 506                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 507                     for k, v in to_send.items():
 508                         self.ns.msg.write(main_topic, k, v)
 509                         return_text += "  {}: {}\n".format(k, v)
 510                 elif cherrypy.request.method == "GET":
 511                     for k, v in kwargs.items():
 512                         self.ns.msg.write(
 513                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 514                         )
 515                         return_text += "  {}: {}\n".format(
 516                             k, yaml.load(v, Loader=yaml.SafeLoader)
 517                         )
 518             except Exception as e:
 519                 return_text += "Error: " + str(e)
 520
 521             return_text += "</pre></html>\n"
 522
 523             return return_text
 524
 525         return_text = (
 526             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 527             + "  kwargs: {}\n".format(kwargs)
 528             + "  headers: {}\n".format(cherrypy.request.headers)
 529             + "  path_info: {}\n".format(cherrypy.request.path_info)
 530             + "  query_string: {}\n".format(cherrypy.request.query_string)
 531             + "  session: {}\n".format(cherrypy.session)
 532             + "  cookie: {}\n".format(cherrypy.request.cookie)
 533             + "  method: {}\n".format(cherrypy.request.method)
 534             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 535             + "  body:\n"
 536         )
 537         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 538
 539         if cherrypy.request.body.length:
 540             return_text += "    content: {}\n".format(
 541                 str(
 542                     cherrypy.request.body.read(
 543                         int(cherrypy.request.headers.get("Content-Length", 0))
 544                     )
 545                 )
 546             )
 547
 548         if thread_info:
 549             return_text += "thread: {}\n".format(thread_info)
 550
 551         return_text += "</pre></html>"
 552
 553         return return_text
 554
 555     @staticmethod
 556     def _check_valid_url_method(method, *args):
 557         if len(args) < 3:
 558             raise RoException(
 559                 "URL must contain at least 'main_topic/version/topic'",
 560                 HTTPStatus.METHOD_NOT_ALLOWED,
 561             )
 562
 563         reference = valid_url_methods
 564         for arg in args:
 565             if arg is None:
 566                 break
 567
 568             if not isinstance(reference, dict):
 569                 raise RoException(
 570                     "URL contains unexpected extra items '{}'".format(arg),
 571                     HTTPStatus.METHOD_NOT_ALLOWED,
 572                 )
 573
 574             if arg in reference:
 575                 reference = reference[arg]
 576             elif "<ID>" in reference:
 577                 reference = reference["<ID>"]
 578             elif "*" in reference:
 579                 # reference = reference["*"]
 580                 break
 581             else:
 582                 raise RoException(
 583                     "Unexpected URL item {}".format(arg),
 584                     HTTPStatus.METHOD_NOT_ALLOWED,
 585                 )
 586
 587         if "TODO" in reference and method in reference["TODO"]:
 588             raise RoException(
 589                 "Method {} not supported yet for this URL".format(method),
 590                 HTTPStatus.NOT_IMPLEMENTED,
 591             )
 592         elif "METHODS" not in reference or method not in reference["METHODS"]:
 593             raise RoException(
 594                 "Method {} not supported for this URL".format(method),
 595                 HTTPStatus.METHOD_NOT_ALLOWED,
 596             )
 597
 598         return reference["ROLE_PERMISSION"] + method.lower()
 599
 600     @staticmethod
 601     def _set_location_header(main_topic, version, topic, id):
 602         """
 603         Insert response header Location with the URL of created item base on URL params
 604         :param main_topic:
 605         :param version:
 606         :param topic:
 607         :param id:
 608         :return: None
 609         """
 610         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 611         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 612             main_topic, version, topic, id
 613         )
 614
 615         return
 616
 617     @cherrypy.expose
 618     def default(
 619         self,
 620         main_topic=None,
 621         version=None,
 622         topic=None,
 623         _id=None,
 624         _id2=None,
 625         *args,
 626         **kwargs,
 627     ):
 628         token_info = None
 629         outdata = None
 630         _format = None
 631         method = "DONE"
 632         rollback = []
 633         engine_session = None
 634
 635         try:
 636             if not main_topic or not version or not topic:
 637                 raise RoException(
 638                     "URL must contain at least 'main_topic/version/topic'",
 639                     HTTPStatus.METHOD_NOT_ALLOWED,
 640                 )
 641
 642             if main_topic not in (
 643                 "admin",
 644                 "ns",
 645             ):
 646                 raise RoException(
 647                     "URL main_topic '{}' not supported".format(main_topic),
 648                     HTTPStatus.METHOD_NOT_ALLOWED,
 649                 )
 650
 651             if version != "v1":
 652                 raise RoException(
 653                     "URL version '{}' not supported".format(version),
 654                     HTTPStatus.METHOD_NOT_ALLOWED,
 655                 )
 656
 657             if (
 658                 kwargs
 659                 and "METHOD" in kwargs
 660                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 661             ):
 662                 method = kwargs.pop("METHOD")
 663             else:
 664                 method = cherrypy.request.method
 665
 666             role_permission = self._check_valid_url_method(
 667                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 668             )
 669             # skip token validation if requesting a token
 670             indata = self._format_in(kwargs)
 671
 672             if main_topic != "admin" or topic != "tokens":
 673                 token_info = self.authenticator.authorize(role_permission, _id)
 674
 675             outdata, created_id, done = self.map_operation[role_permission](
 676                 engine_session, indata, version, _id, _id2, *args, *kwargs
 677             )
 678
 679             if created_id:
 680                 self._set_location_header(main_topic, version, topic, _id)
 681
 682             cherrypy.response.status = (
 683                 HTTPStatus.ACCEPTED.value
 684                 if not done
 685                 else HTTPStatus.OK.value
 686                 if outdata is not None
 687                 else HTTPStatus.NO_CONTENT.value
 688             )
 689
 690             return self._format_out(outdata, token_info, _format)
 691         except Exception as e:
 692             if isinstance(
 693                 e,
 694                 (
 695                     RoException,
 696                     NsException,
 697                     DbException,
 698                     FsException,
 699                     MsgException,
 700                     AuthException,
 701                     ValidationError,
 702                 ),
 703             ):
 704                 http_code_value = cherrypy.response.status = e.http_code.value
 705                 http_code_name = e.http_code.name
 706                 cherrypy.log("Exception {}".format(e))
 707             else:
 708                 http_code_value = (
 709                     cherrypy.response.status
 710                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 711                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 712                 http_code_name = HTTPStatus.BAD_REQUEST.name
 713
 714             if hasattr(outdata, "close"):  # is an open file
 715                 outdata.close()
 716
 717             error_text = str(e)
 718             rollback.reverse()
 719
 720             for rollback_item in rollback:
 721                 try:
 722                     if rollback_item.get("operation") == "set":
 723                         self.ns.db.set_one(
 724                             rollback_item["topic"],
 725                             {"_id": rollback_item["_id"]},
 726                             rollback_item["content"],
 727                             fail_on_empty=False,
 728                         )
 729                     else:
 730                         self.ns.db.del_one(
 731                             rollback_item["topic"],
 732                             {"_id": rollback_item["_id"]},
 733                             fail_on_empty=False,
 734                         )
 735                 except Exception as e2:
 736                     rollback_error_text = "Rollback Exception {}: {}".format(
 737                         rollback_item, e2
 738                     )
 739                     cherrypy.log(rollback_error_text)
 740                     error_text += ". " + rollback_error_text
 741
 742             # if isinstance(e, MsgException):
 743             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 744             #         engine_topic[:-1], method, error_text)
 745             problem_details = {
 746                 "code": http_code_name,
 747                 "status": http_code_value,
 748                 "detail": error_text,
 749             }
 750
 751             return self._format_out(problem_details, token_info)
 752             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 753         finally:
 754             if token_info:
 755                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 756                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 757                         if outdata.get(logging_id):
 758                             cherrypy.request.login += ";{}={}".format(
 759                                 logging_id, outdata[logging_id][:36]
 760                             )
 761
 762
 763 def _start_service():
 764     """
 765     Callback function called when cherrypy.engine starts
 766     Override configuration with env variables
 767     Set database, storage, message configuration
 768     Init database with admin/admin user password
 769     """
 770     global ro_server, vim_admin_thread
 771     # global vim_threads
 772     cherrypy.log.error("Starting osm_ng_ro")
 773     # update general cherrypy configuration
 774     update_dict = {}
 775     engine_config = cherrypy.tree.apps["/ro"].config
 776
 777     for k, v in environ.items():
 778         if not k.startswith("OSMRO_"):
 779             continue
 780
 781         k1, _, k2 = k[6:].lower().partition("_")
 782
 783         if not k2:
 784             continue
 785
 786         try:
 787             if k1 in ("server", "test", "auth", "log"):
 788                 # update [global] configuration
 789                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 790             elif k1 == "static":
 791                 # update [/static] configuration
 792                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 793             elif k1 == "tools":
 794                 # update [/] configuration
 795                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 796             elif k1 in ("message", "database", "storage", "authentication"):
 797                 engine_config[k1][k2] = yaml.safe_load(v)
 798
 799         except Exception as e:
 800             raise RoException("Cannot load env '{}': {}".format(k, e))
 801
 802     if update_dict:
 803         cherrypy.config.update(update_dict)
 804         engine_config["global"].update(update_dict)
 805
 806     # logging cherrypy
 807     log_format_simple = (
 808         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 809     )
 810     log_formatter_simple = logging.Formatter(
 811         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 812     )
 813     logger_server = logging.getLogger("cherrypy.error")
 814     logger_access = logging.getLogger("cherrypy.access")
 815     logger_cherry = logging.getLogger("cherrypy")
 816     logger = logging.getLogger("ro")
 817
 818     if "log.file" in engine_config["global"]:
 819         file_handler = logging.handlers.RotatingFileHandler(
 820             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 821         )
 822         file_handler.setFormatter(log_formatter_simple)
 823         logger_cherry.addHandler(file_handler)
 824         logger.addHandler(file_handler)
 825
 826     # log always to standard output
 827     for format_, logger in {
 828         "ro.server %(filename)s:%(lineno)s": logger_server,
 829         "ro.access %(filename)s:%(lineno)s": logger_access,
 830         "%(name)s %(filename)s:%(lineno)s": logger,
 831     }.items():
 832         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 833         log_formatter_cherry = logging.Formatter(
 834             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 835         )
 836         str_handler = logging.StreamHandler()
 837         str_handler.setFormatter(log_formatter_cherry)
 838         logger.addHandler(str_handler)
 839
 840     if engine_config["global"].get("log.level"):
 841         logger_cherry.setLevel(engine_config["global"]["log.level"])
 842         logger.setLevel(engine_config["global"]["log.level"])
 843
 844     # logging other modules
 845     for k1, logname in {
 846         "message": "ro.msg",
 847         "database": "ro.db",
 848         "storage": "ro.fs",
 849     }.items():
 850         engine_config[k1]["logger_name"] = logname
 851         logger_module = logging.getLogger(logname)
 852
 853         if "logfile" in engine_config[k1]:
 854             file_handler = logging.handlers.RotatingFileHandler(
 855                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 856             )
 857             file_handler.setFormatter(log_formatter_simple)
 858             logger_module.addHandler(file_handler)
 859
 860         if "loglevel" in engine_config[k1]:
 861             logger_module.setLevel(engine_config[k1]["loglevel"])
 862     # TODO add more entries, e.g.: storage
 863
 864     engine_config["assignment"] = {}
 865     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 866     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 867     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 868     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 869
 870     # # start subscriptions thread:
 871     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 872     vim_admin_thread.start()
 873     # # Do not capture except SubscriptionException
 874
 875     # backend = engine_config["authentication"]["backend"]
 876     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 877     #                    .format(ro_version, ro_version_date, backend))
 878
 879
 880 def _stop_service():
 881     """
 882     Callback function called when cherrypy.engine stops
 883     TODO: Ending database connections.
 884     """
 885     global vim_admin_thread
 886
 887     # terminate vim_admin_thread
 888     if vim_admin_thread:
 889         vim_admin_thread.terminate()
 890
 891     vim_admin_thread = None
 892     cherrypy.tree.apps["/ro"].root.ns.stop()
 893     cherrypy.log.error("Stopping osm_ng_ro")
 894
 895
 896 def ro_main(config_file):
 897     global ro_server
 898
 899     ro_server = Server()
 900     cherrypy.engine.subscribe("start", _start_service)
 901     cherrypy.engine.subscribe("stop", _stop_service)
 902     cherrypy.quickstart(ro_server, "/ro", config_file)
 903
 904
 905 def usage():
 906     print(
 907         """Usage: {} [options]
 908         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 909         -h|--help: shows this help
 910         """.format(
 911             sys.argv[0]
 912         )
 913     )
 914     # --log-socket-host HOST: send logs to this host")
 915     # --log-socket-port PORT: send logs using this port (default: 9022)")
 916
 917
 918 if __name__ == "__main__":
 919     try:
 920         # load parameters and configuration
 921         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 922         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 923         config_file = None
 924
 925         for o, a in opts:
 926             if o in ("-h", "--help"):
 927                 usage()
 928                 sys.exit()
 929             elif o in ("-c", "--config"):
 930                 config_file = a
 931             else:
 932                 assert False, "Unhandled option"
 933
 934         if config_file:
 935             if not path.isfile(config_file):
 936                 print(
 937                     "configuration file '{}' that not exist".format(config_file),
 938                     file=sys.stderr,
 939                 )
 940                 exit(1)
 941         else:
 942             for config_file in (
 943                 path.dirname(__file__) + "/ro.cfg",
 944                 "./ro.cfg",
 945                 "/etc/osm/ro.cfg",
 946             ):
 947                 if path.isfile(config_file):
 948                     break
 949             else:
 950                 print(
 951                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 952                     file=sys.stderr,
 953                 )
 954                 exit(1)
 955
 956         ro_main(config_file)
 957     except KeyboardInterrupt:
 958         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 959     except getopt.GetoptError as e:
 960         print(str(e), file=sys.stderr)
 961         # usage()
 962         exit(1)