NG-RO/osm_ng_ro/ro_main.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  16 # implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 ##
  20
  21
  22 from codecs import getreader
  23 import getopt
  24 from http import HTTPStatus
  25 import json
  26 import logging
  27 import logging.handlers
  28 from os import environ, path
  29 import sys
  30 import time
  31
  32 import cherrypy
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35 from osm_common.msgbase import MsgException
  36 from osm_ng_ro import version as ro_version, version_date as ro_version_date
  37 import osm_ng_ro.html_out as html
  38 from osm_ng_ro.ns import Ns, NsException
  39 from osm_ng_ro.validation import ValidationError
  40 from osm_ng_ro.vim_admin import VimAdminThread
  41 import yaml
  42
  43
  44 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  45 __version__ = "0.1."  # file version, not NBI version
  46 version_date = "May 2020"
  47
  48 database_version = "1.2"
  49 auth_database_version = "1.0"
  50 ro_server = None  # instance of Server class
  51 vim_admin_thread = None  # instance of VimAdminThread class
  52
  53 # vim_threads = None  # instance of VimThread class
  54
  55 """
  56 RO North Bound Interface
  57 URL: /ro                                                       GET     POST    PUT     DELETE  PATCH
  58         /ns/v1/deploy                                           O
  59             /<nsrs_id>                                          O       O               O
  60                 /<action_id>                                    O
  61                     /cancel                                             O
  62
  63 """
  64
  65 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
  66 # ^ Contains possible administrative query string words:
  67 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
  68 #           (not owned by my session project).
  69 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
  70 #     FORCE=True(by default)|False: Force edition/deletion operations
  71 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
  72
  73 valid_url_methods = {
  74     # contains allowed URL and methods, and the role_permission name
  75     "admin": {
  76         "v1": {
  77             "tokens": {
  78                 "METHODS": ("POST",),
  79                 "ROLE_PERMISSION": "tokens:",
  80                 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
  81             },
  82         }
  83     },
  84     "ns": {
  85         "v1": {
  86             "deploy": {
  87                 "METHODS": ("GET",),
  88                 "ROLE_PERMISSION": "deploy:",
  89                 "<ID>": {
  90                     "METHODS": ("GET", "POST", "DELETE"),
  91                     "ROLE_PERMISSION": "deploy:id:",
  92                     "<ID>": {
  93                         "METHODS": ("GET",),
  94                         "ROLE_PERMISSION": "deploy:id:id:",
  95                         "cancel": {
  96                             "METHODS": ("POST",),
  97                             "ROLE_PERMISSION": "deploy:id:id:cancel",
  98                         },
  99                     },
 100                 },
 101             },
 102             "recreate": {
 103                 "<ID>": {
 104                     "METHODS": ("POST"),
 105                     "ROLE_PERMISSION": "recreate:id:",
 106                     "<ID>": {
 107                         "METHODS": ("GET",),
 108                         "ROLE_PERMISSION": "recreate:id:id:",
 109                     },
 110                 },
 111             },
 112             "migrate": {
 113                 "<ID>": {
 114                     "METHODS": ("POST"),
 115                     "ROLE_PERMISSION": "migrate:id:",
 116                     "<ID>": {
 117                         "METHODS": ("GET",),
 118                         "ROLE_PERMISSION": "migrate:id:id:",
 119                     },
 120                 },
 121             },
 122         }
 123     },
 124 }
 125
 126
 127 class RoException(Exception):
 128     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 129         Exception.__init__(self, message)
 130         self.http_code = http_code
 131
 132
 133 class AuthException(RoException):
 134     pass
 135
 136
 137 class Authenticator:
 138     def __init__(self, valid_url_methods, valid_query_string):
 139         self.valid_url_methods = valid_url_methods
 140         self.valid_query_string = valid_query_string
 141
 142     def authorize(self, *args, **kwargs):
 143         return {"token": "ok", "id": "ok"}
 144
 145     def new_token(self, token_info, indata, remote):
 146         return {"token": "ok", "id": "ok", "remote": remote}
 147
 148     def del_token(self, token_id):
 149         pass
 150
 151     def start(self, engine_config):
 152         pass
 153
 154
 155 class Server(object):
 156     instance = 0
 157     # to decode bytes to str
 158     reader = getreader("utf-8")
 159
 160     def __init__(self):
 161         self.instance += 1
 162         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 163         self.ns = Ns()
 164         self.map_operation = {
 165             "token:post": self.new_token,
 166             "token:id:delete": self.del_token,
 167             "deploy:get": self.ns.get_deploy,
 168             "deploy:id:get": self.ns.get_actions,
 169             "deploy:id:post": self.ns.deploy,
 170             "deploy:id:delete": self.ns.delete,
 171             "deploy:id:id:get": self.ns.status,
 172             "deploy:id:id:cancel:post": self.ns.cancel,
 173             "recreate:id:post": self.ns.recreate,
 174             "recreate:id:id:get": self.ns.recreate_status,
 175             "migrate:id:post": self.ns.migrate,
 176         }
 177
 178     def _format_in(self, kwargs):
 179         try:
 180             indata = None
 181
 182             if cherrypy.request.body.length:
 183                 error_text = "Invalid input format "
 184
 185                 if "Content-Type" in cherrypy.request.headers:
 186                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 187                         error_text = "Invalid json format "
 188                         indata = json.load(self.reader(cherrypy.request.body))
 189                         cherrypy.request.headers.pop("Content-File-MD5", None)
 190                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 191                         error_text = "Invalid yaml format "
 192                         indata = yaml.load(
 193                             cherrypy.request.body, Loader=yaml.SafeLoader
 194                         )
 195                         cherrypy.request.headers.pop("Content-File-MD5", None)
 196                     elif (
 197                         "application/binary" in cherrypy.request.headers["Content-Type"]
 198                         or "application/gzip"
 199                         in cherrypy.request.headers["Content-Type"]
 200                         or "application/zip" in cherrypy.request.headers["Content-Type"]
 201                         or "text/plain" in cherrypy.request.headers["Content-Type"]
 202                     ):
 203                         indata = cherrypy.request.body  # .read()
 204                     elif (
 205                         "multipart/form-data"
 206                         in cherrypy.request.headers["Content-Type"]
 207                     ):
 208                         if "descriptor_file" in kwargs:
 209                             filecontent = kwargs.pop("descriptor_file")
 210
 211                             if not filecontent.file:
 212                                 raise RoException(
 213                                     "empty file or content", HTTPStatus.BAD_REQUEST
 214                                 )
 215
 216                             indata = filecontent.file  # .read()
 217
 218                             if filecontent.content_type.value:
 219                                 cherrypy.request.headers[
 220                                     "Content-Type"
 221                                 ] = filecontent.content_type.value
 222                     else:
 223                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 224                         #                          "Only 'Content-Type' of type 'application/json' or
 225                         # 'application/yaml' for input format are available")
 226                         error_text = "Invalid yaml format "
 227                         indata = yaml.load(
 228                             cherrypy.request.body, Loader=yaml.SafeLoader
 229                         )
 230                         cherrypy.request.headers.pop("Content-File-MD5", None)
 231                 else:
 232                     error_text = "Invalid yaml format "
 233                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 234                     cherrypy.request.headers.pop("Content-File-MD5", None)
 235
 236             if not indata:
 237                 indata = {}
 238
 239             format_yaml = False
 240             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 241                 format_yaml = True
 242
 243             for k, v in kwargs.items():
 244                 if isinstance(v, str):
 245                     if v == "":
 246                         kwargs[k] = None
 247                     elif format_yaml:
 248                         try:
 249                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 250                         except Exception:
 251                             pass
 252                     elif (
 253                         k.endswith(".gt")
 254                         or k.endswith(".lt")
 255                         or k.endswith(".gte")
 256                         or k.endswith(".lte")
 257                     ):
 258                         try:
 259                             kwargs[k] = int(v)
 260                         except Exception:
 261                             try:
 262                                 kwargs[k] = float(v)
 263                             except Exception:
 264                                 pass
 265                     elif v.find(",") > 0:
 266                         kwargs[k] = v.split(",")
 267                 elif isinstance(v, (list, tuple)):
 268                     for index in range(0, len(v)):
 269                         if v[index] == "":
 270                             v[index] = None
 271                         elif format_yaml:
 272                             try:
 273                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 274                             except Exception:
 275                                 pass
 276
 277             return indata
 278         except (ValueError, yaml.YAMLError) as exc:
 279             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 280         except KeyError as exc:
 281             raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 282         except Exception as exc:
 283             raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 284
 285     @staticmethod
 286     def _format_out(data, token_info=None, _format=None):
 287         """
 288         return string of dictionary data according to requested json, yaml, xml. By default json
 289         :param data: response to be sent. Can be a dict, text or file
 290         :param token_info: Contains among other username and project
 291         :param _format: The format to be set as Content-Type if data is a file
 292         :return: None
 293         """
 294         accept = cherrypy.request.headers.get("Accept")
 295
 296         if data is None:
 297             if accept and "text/html" in accept:
 298                 return html.format(
 299                     data, cherrypy.request, cherrypy.response, token_info
 300                 )
 301
 302             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 303             return
 304         elif hasattr(data, "read"):  # file object
 305             if _format:
 306                 cherrypy.response.headers["Content-Type"] = _format
 307             elif "b" in data.mode:  # binariy asssumig zip
 308                 cherrypy.response.headers["Content-Type"] = "application/zip"
 309             else:
 310                 cherrypy.response.headers["Content-Type"] = "text/plain"
 311
 312             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 313             return data
 314
 315         if accept:
 316             if "application/json" in accept:
 317                 cherrypy.response.headers[
 318                     "Content-Type"
 319                 ] = "application/json; charset=utf-8"
 320                 a = json.dumps(data, indent=4) + "\n"
 321
 322                 return a.encode("utf8")
 323             elif "text/html" in accept:
 324                 return html.format(
 325                     data, cherrypy.request, cherrypy.response, token_info
 326                 )
 327             elif (
 328                 "application/yaml" in accept
 329                 or "*/*" in accept
 330                 or "text/plain" in accept
 331             ):
 332                 pass
 333             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 334             elif cherrypy.response.status >= 400:
 335                 raise cherrypy.HTTPError(
 336                     HTTPStatus.NOT_ACCEPTABLE.value,
 337                     "Only 'Accept' of type 'application/json' or 'application/yaml' "
 338                     "for output format are available",
 339                 )
 340
 341         cherrypy.response.headers["Content-Type"] = "application/yaml"
 342
 343         return yaml.safe_dump(
 344             data,
 345             explicit_start=True,
 346             indent=4,
 347             default_flow_style=False,
 348             tags=False,
 349             encoding="utf-8",
 350             allow_unicode=True,
 351         )  # , canonical=True, default_style='"'
 352
 353     @cherrypy.expose
 354     def index(self, *args, **kwargs):
 355         token_info = None
 356
 357         try:
 358             if cherrypy.request.method == "GET":
 359                 token_info = self.authenticator.authorize()
 360                 outdata = token_info  # Home page
 361             else:
 362                 raise cherrypy.HTTPError(
 363                     HTTPStatus.METHOD_NOT_ALLOWED.value,
 364                     "Method {} not allowed for tokens".format(cherrypy.request.method),
 365                 )
 366
 367             return self._format_out(outdata, token_info)
 368         except (NsException, AuthException) as e:
 369             # cherrypy.log("index Exception {}".format(e))
 370             cherrypy.response.status = e.http_code.value
 371
 372             return self._format_out("Welcome to OSM!", token_info)
 373
 374     @cherrypy.expose
 375     def version(self, *args, **kwargs):
 376         # TODO consider to remove and provide version using the static version file
 377         try:
 378             if cherrypy.request.method != "GET":
 379                 raise RoException(
 380                     "Only method GET is allowed",
 381                     HTTPStatus.METHOD_NOT_ALLOWED,
 382                 )
 383             elif args or kwargs:
 384                 raise RoException(
 385                     "Invalid URL or query string for version",
 386                     HTTPStatus.METHOD_NOT_ALLOWED,
 387                 )
 388
 389             # TODO include version of other modules, pick up from some kafka admin message
 390             osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
 391
 392             return self._format_out(osm_ng_ro_version)
 393         except RoException as e:
 394             cherrypy.response.status = e.http_code.value
 395             problem_details = {
 396                 "code": e.http_code.name,
 397                 "status": e.http_code.value,
 398                 "detail": str(e),
 399             }
 400
 401             return self._format_out(problem_details, None)
 402
 403     def new_token(self, engine_session, indata, *args, **kwargs):
 404         token_info = None
 405
 406         try:
 407             token_info = self.authenticator.authorize()
 408         except Exception:
 409             token_info = None
 410
 411         if kwargs:
 412             indata.update(kwargs)
 413
 414         # This is needed to log the user when authentication fails
 415         cherrypy.request.login = "{}".format(indata.get("username", "-"))
 416         token_info = self.authenticator.new_token(
 417             token_info, indata, cherrypy.request.remote
 418         )
 419         cherrypy.session["Authorization"] = token_info["id"]
 420         self._set_location_header("admin", "v1", "tokens", token_info["id"])
 421         # for logging
 422
 423         # cherrypy.response.cookie["Authorization"] = outdata["id"]
 424         # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 425
 426         return token_info, token_info["id"], True
 427
 428     def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
 429         token_id = _id
 430
 431         if not token_id and "id" in kwargs:
 432             token_id = kwargs["id"]
 433         elif not token_id:
 434             token_info = self.authenticator.authorize()
 435             # for logging
 436             token_id = token_info["id"]
 437
 438         self.authenticator.del_token(token_id)
 439         token_info = None
 440         cherrypy.session["Authorization"] = "logout"
 441         # cherrypy.response.cookie["Authorization"] = token_id
 442         # cherrypy.response.cookie["Authorization"]['expires'] = 0
 443
 444         return None, None, True
 445
 446     @cherrypy.expose
 447     def test(self, *args, **kwargs):
 448         if not cherrypy.config.get("server.enable_test") or (
 449             isinstance(cherrypy.config["server.enable_test"], str)
 450             and cherrypy.config["server.enable_test"].lower() == "false"
 451         ):
 452             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 453
 454             return "test URL is disabled"
 455
 456         thread_info = None
 457
 458         if args and args[0] == "help":
 459             return (
 460                 "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
 461                 "sleep/<time>\nmessage/topic\n</pre></html>"
 462             )
 463         elif args and args[0] == "init":
 464             try:
 465                 # self.ns.load_dbase(cherrypy.request.app.config)
 466                 self.ns.create_admin()
 467
 468                 return "Done. User 'admin', password 'admin' created"
 469             except Exception:
 470                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 471
 472                 return self._format_out("Database already initialized")
 473         elif args and args[0] == "file":
 474             return cherrypy.lib.static.serve_file(
 475                 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
 476                 "text/plain",
 477                 "attachment",
 478             )
 479         elif args and args[0] == "file2":
 480             f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
 481             f = open(f_path, "r")
 482             cherrypy.response.headers["Content-type"] = "text/plain"
 483             return f
 484
 485         elif len(args) == 2 and args[0] == "db-clear":
 486             deleted_info = self.ns.db.del_list(args[1], kwargs)
 487             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 488         elif len(args) and args[0] == "fs-clear":
 489             if len(args) >= 2:
 490                 folders = (args[1],)
 491             else:
 492                 folders = self.ns.fs.dir_ls(".")
 493
 494             for folder in folders:
 495                 self.ns.fs.file_delete(folder)
 496
 497             return ",".join(folders) + " folders deleted\n"
 498         elif args and args[0] == "login":
 499             if not cherrypy.request.headers.get("Authorization"):
 500                 cherrypy.response.headers[
 501                     "WWW-Authenticate"
 502                 ] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 503                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 504         elif args and args[0] == "login2":
 505             if not cherrypy.request.headers.get("Authorization"):
 506                 cherrypy.response.headers[
 507                     "WWW-Authenticate"
 508                 ] = 'Bearer realm="Access to OSM site"'
 509                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 510         elif args and args[0] == "sleep":
 511             sleep_time = 5
 512
 513             try:
 514                 sleep_time = int(args[1])
 515             except Exception:
 516                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 517                 return self._format_out("Database already initialized")
 518
 519             thread_info = cherrypy.thread_data
 520             print(thread_info)
 521             time.sleep(sleep_time)
 522             # thread_info
 523         elif len(args) >= 2 and args[0] == "message":
 524             main_topic = args[1]
 525             return_text = "<html><pre>{} ->\n".format(main_topic)
 526
 527             try:
 528                 if cherrypy.request.method == "POST":
 529                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 530                     for k, v in to_send.items():
 531                         self.ns.msg.write(main_topic, k, v)
 532                         return_text += "  {}: {}\n".format(k, v)
 533                 elif cherrypy.request.method == "GET":
 534                     for k, v in kwargs.items():
 535                         self.ns.msg.write(
 536                             main_topic, k, yaml.load(v, Loader=yaml.SafeLoader)
 537                         )
 538                         return_text += "  {}: {}\n".format(
 539                             k, yaml.load(v, Loader=yaml.SafeLoader)
 540                         )
 541             except Exception as e:
 542                 return_text += "Error: " + str(e)
 543
 544             return_text += "</pre></html>\n"
 545
 546             return return_text
 547
 548         return_text = (
 549             "<html><pre>\nheaders:\n  args: {}\n".format(args)
 550             + "  kwargs: {}\n".format(kwargs)
 551             + "  headers: {}\n".format(cherrypy.request.headers)
 552             + "  path_info: {}\n".format(cherrypy.request.path_info)
 553             + "  query_string: {}\n".format(cherrypy.request.query_string)
 554             + "  session: {}\n".format(cherrypy.session)
 555             + "  cookie: {}\n".format(cherrypy.request.cookie)
 556             + "  method: {}\n".format(cherrypy.request.method)
 557             + "  session: {}\n".format(cherrypy.session.get("fieldname"))
 558             + "  body:\n"
 559         )
 560         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 561
 562         if cherrypy.request.body.length:
 563             return_text += "    content: {}\n".format(
 564                 str(
 565                     cherrypy.request.body.read(
 566                         int(cherrypy.request.headers.get("Content-Length", 0))
 567                     )
 568                 )
 569             )
 570
 571         if thread_info:
 572             return_text += "thread: {}\n".format(thread_info)
 573
 574         return_text += "</pre></html>"
 575
 576         return return_text
 577
 578     @staticmethod
 579     def _check_valid_url_method(method, *args):
 580         if len(args) < 3:
 581             raise RoException(
 582                 "URL must contain at least 'main_topic/version/topic'",
 583                 HTTPStatus.METHOD_NOT_ALLOWED,
 584             )
 585
 586         reference = valid_url_methods
 587         for arg in args:
 588             if arg is None:
 589                 break
 590
 591             if not isinstance(reference, dict):
 592                 raise RoException(
 593                     "URL contains unexpected extra items '{}'".format(arg),
 594                     HTTPStatus.METHOD_NOT_ALLOWED,
 595                 )
 596
 597             if arg in reference:
 598                 reference = reference[arg]
 599             elif "<ID>" in reference:
 600                 reference = reference["<ID>"]
 601             elif "*" in reference:
 602                 # reference = reference["*"]
 603                 break
 604             else:
 605                 raise RoException(
 606                     "Unexpected URL item {}".format(arg),
 607                     HTTPStatus.METHOD_NOT_ALLOWED,
 608                 )
 609
 610         if "TODO" in reference and method in reference["TODO"]:
 611             raise RoException(
 612                 "Method {} not supported yet for this URL".format(method),
 613                 HTTPStatus.NOT_IMPLEMENTED,
 614             )
 615         elif "METHODS" not in reference or method not in reference["METHODS"]:
 616             raise RoException(
 617                 "Method {} not supported for this URL".format(method),
 618                 HTTPStatus.METHOD_NOT_ALLOWED,
 619             )
 620
 621         return reference["ROLE_PERMISSION"] + method.lower()
 622
 623     @staticmethod
 624     def _set_location_header(main_topic, version, topic, id):
 625         """
 626         Insert response header Location with the URL of created item base on URL params
 627         :param main_topic:
 628         :param version:
 629         :param topic:
 630         :param id:
 631         :return: None
 632         """
 633         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 634         cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
 635             main_topic, version, topic, id
 636         )
 637
 638         return
 639
 640     @cherrypy.expose
 641     def default(
 642         self,
 643         main_topic=None,
 644         version=None,
 645         topic=None,
 646         _id=None,
 647         _id2=None,
 648         *args,
 649         **kwargs,
 650     ):
 651         token_info = None
 652         outdata = None
 653         _format = None
 654         method = "DONE"
 655         rollback = []
 656         engine_session = None
 657
 658         try:
 659             if not main_topic or not version or not topic:
 660                 raise RoException(
 661                     "URL must contain at least 'main_topic/version/topic'",
 662                     HTTPStatus.METHOD_NOT_ALLOWED,
 663                 )
 664
 665             if main_topic not in (
 666                 "admin",
 667                 "ns",
 668             ):
 669                 raise RoException(
 670                     "URL main_topic '{}' not supported".format(main_topic),
 671                     HTTPStatus.METHOD_NOT_ALLOWED,
 672                 )
 673
 674             if version != "v1":
 675                 raise RoException(
 676                     "URL version '{}' not supported".format(version),
 677                     HTTPStatus.METHOD_NOT_ALLOWED,
 678                 )
 679
 680             if (
 681                 kwargs
 682                 and "METHOD" in kwargs
 683                 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
 684             ):
 685                 method = kwargs.pop("METHOD")
 686             else:
 687                 method = cherrypy.request.method
 688
 689             role_permission = self._check_valid_url_method(
 690                 method, main_topic, version, topic, _id, _id2, *args, **kwargs
 691             )
 692             # skip token validation if requesting a token
 693             indata = self._format_in(kwargs)
 694
 695             if main_topic != "admin" or topic != "tokens":
 696                 token_info = self.authenticator.authorize(role_permission, _id)
 697
 698             outdata, created_id, done = self.map_operation[role_permission](
 699                 engine_session, indata, version, _id, _id2, *args, *kwargs
 700             )
 701
 702             if created_id:
 703                 self._set_location_header(main_topic, version, topic, _id)
 704
 705             cherrypy.response.status = (
 706                 HTTPStatus.ACCEPTED.value
 707                 if not done
 708                 else HTTPStatus.OK.value
 709                 if outdata is not None
 710                 else HTTPStatus.NO_CONTENT.value
 711             )
 712
 713             return self._format_out(outdata, token_info, _format)
 714         except Exception as e:
 715             if isinstance(
 716                 e,
 717                 (
 718                     RoException,
 719                     NsException,
 720                     DbException,
 721                     FsException,
 722                     MsgException,
 723                     AuthException,
 724                     ValidationError,
 725                 ),
 726             ):
 727                 http_code_value = cherrypy.response.status = e.http_code.value
 728                 http_code_name = e.http_code.name
 729                 cherrypy.log("Exception {}".format(e))
 730             else:
 731                 http_code_value = (
 732                     cherrypy.response.status
 733                 ) = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 734                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
 735                 http_code_name = HTTPStatus.BAD_REQUEST.name
 736
 737             if hasattr(outdata, "close"):  # is an open file
 738                 outdata.close()
 739
 740             error_text = str(e)
 741             rollback.reverse()
 742
 743             for rollback_item in rollback:
 744                 try:
 745                     if rollback_item.get("operation") == "set":
 746                         self.ns.db.set_one(
 747                             rollback_item["topic"],
 748                             {"_id": rollback_item["_id"]},
 749                             rollback_item["content"],
 750                             fail_on_empty=False,
 751                         )
 752                     else:
 753                         self.ns.db.del_one(
 754                             rollback_item["topic"],
 755                             {"_id": rollback_item["_id"]},
 756                             fail_on_empty=False,
 757                         )
 758                 except Exception as e2:
 759                     rollback_error_text = "Rollback Exception {}: {}".format(
 760                         rollback_item, e2
 761                     )
 762                     cherrypy.log(rollback_error_text)
 763                     error_text += ". " + rollback_error_text
 764
 765             # if isinstance(e, MsgException):
 766             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 767             #         engine_topic[:-1], method, error_text)
 768             problem_details = {
 769                 "code": http_code_name,
 770                 "status": http_code_value,
 771                 "detail": error_text,
 772             }
 773
 774             return self._format_out(problem_details, token_info)
 775             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 776         finally:
 777             if token_info:
 778                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
 779                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
 780                         if outdata.get(logging_id):
 781                             cherrypy.request.login += ";{}={}".format(
 782                                 logging_id, outdata[logging_id][:36]
 783                             )
 784
 785
 786 def _start_service():
 787     """
 788     Callback function called when cherrypy.engine starts
 789     Override configuration with env variables
 790     Set database, storage, message configuration
 791     Init database with admin/admin user password
 792     """
 793     global ro_server, vim_admin_thread
 794     # global vim_threads
 795     cherrypy.log.error("Starting osm_ng_ro")
 796     # update general cherrypy configuration
 797     update_dict = {}
 798     engine_config = cherrypy.tree.apps["/ro"].config
 799
 800     for k, v in environ.items():
 801         if not k.startswith("OSMRO_"):
 802             continue
 803
 804         k1, _, k2 = k[6:].lower().partition("_")
 805
 806         if not k2:
 807             continue
 808
 809         try:
 810             if k1 in ("server", "test", "auth", "log"):
 811                 # update [global] configuration
 812                 update_dict[k1 + "." + k2] = yaml.safe_load(v)
 813             elif k1 == "static":
 814                 # update [/static] configuration
 815                 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
 816             elif k1 == "tools":
 817                 # update [/] configuration
 818                 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
 819             elif k1 in ("message", "database", "storage", "authentication"):
 820                 engine_config[k1][k2] = yaml.safe_load(v)
 821
 822         except Exception as e:
 823             raise RoException("Cannot load env '{}': {}".format(k, e))
 824
 825     if update_dict:
 826         cherrypy.config.update(update_dict)
 827         engine_config["global"].update(update_dict)
 828
 829     # logging cherrypy
 830     log_format_simple = (
 831         "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 832     )
 833     log_formatter_simple = logging.Formatter(
 834         log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
 835     )
 836     logger_server = logging.getLogger("cherrypy.error")
 837     logger_access = logging.getLogger("cherrypy.access")
 838     logger_cherry = logging.getLogger("cherrypy")
 839     logger = logging.getLogger("ro")
 840
 841     if "log.file" in engine_config["global"]:
 842         file_handler = logging.handlers.RotatingFileHandler(
 843             engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
 844         )
 845         file_handler.setFormatter(log_formatter_simple)
 846         logger_cherry.addHandler(file_handler)
 847         logger.addHandler(file_handler)
 848
 849     # log always to standard output
 850     for format_, logger in {
 851         "ro.server %(filename)s:%(lineno)s": logger_server,
 852         "ro.access %(filename)s:%(lineno)s": logger_access,
 853         "%(name)s %(filename)s:%(lineno)s": logger,
 854     }.items():
 855         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 856         log_formatter_cherry = logging.Formatter(
 857             log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
 858         )
 859         str_handler = logging.StreamHandler()
 860         str_handler.setFormatter(log_formatter_cherry)
 861         logger.addHandler(str_handler)
 862
 863     if engine_config["global"].get("log.level"):
 864         logger_cherry.setLevel(engine_config["global"]["log.level"])
 865         logger.setLevel(engine_config["global"]["log.level"])
 866
 867     # logging other modules
 868     for k1, logname in {
 869         "message": "ro.msg",
 870         "database": "ro.db",
 871         "storage": "ro.fs",
 872     }.items():
 873         engine_config[k1]["logger_name"] = logname
 874         logger_module = logging.getLogger(logname)
 875
 876         if "logfile" in engine_config[k1]:
 877             file_handler = logging.handlers.RotatingFileHandler(
 878                 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
 879             )
 880             file_handler.setFormatter(log_formatter_simple)
 881             logger_module.addHandler(file_handler)
 882
 883         if "loglevel" in engine_config[k1]:
 884             logger_module.setLevel(engine_config[k1]["loglevel"])
 885     # TODO add more entries, e.g.: storage
 886
 887     engine_config["assignment"] = {}
 888     # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
 889     cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
 890     cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
 891     cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
 892
 893     # # start subscriptions thread:
 894     vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
 895     vim_admin_thread.start()
 896     # # Do not capture except SubscriptionException
 897
 898     # backend = engine_config["authentication"]["backend"]
 899     # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
 900     #                    .format(ro_version, ro_version_date, backend))
 901
 902
 903 def _stop_service():
 904     """
 905     Callback function called when cherrypy.engine stops
 906     TODO: Ending database connections.
 907     """
 908     global vim_admin_thread
 909
 910     # terminate vim_admin_thread
 911     if vim_admin_thread:
 912         vim_admin_thread.terminate()
 913
 914     vim_admin_thread = None
 915     cherrypy.tree.apps["/ro"].root.ns.stop()
 916     cherrypy.log.error("Stopping osm_ng_ro")
 917
 918
 919 def ro_main(config_file):
 920     global ro_server
 921
 922     ro_server = Server()
 923     cherrypy.engine.subscribe("start", _start_service)
 924     cherrypy.engine.subscribe("stop", _stop_service)
 925     cherrypy.quickstart(ro_server, "/ro", config_file)
 926
 927
 928 def usage():
 929     print(
 930         """Usage: {} [options]
 931         -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
 932         -h|--help: shows this help
 933         """.format(
 934             sys.argv[0]
 935         )
 936     )
 937     # --log-socket-host HOST: send logs to this host")
 938     # --log-socket-port PORT: send logs using this port (default: 9022)")
 939
 940
 941 if __name__ == "__main__":
 942     try:
 943         # load parameters and configuration
 944         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 945         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 946         config_file = None
 947
 948         for o, a in opts:
 949             if o in ("-h", "--help"):
 950                 usage()
 951                 sys.exit()
 952             elif o in ("-c", "--config"):
 953                 config_file = a
 954             else:
 955                 assert False, "Unhandled option"
 956
 957         if config_file:
 958             if not path.isfile(config_file):
 959                 print(
 960                     "configuration file '{}' that not exist".format(config_file),
 961                     file=sys.stderr,
 962                 )
 963                 exit(1)
 964         else:
 965             for config_file in (
 966                 path.dirname(__file__) + "/ro.cfg",
 967                 "./ro.cfg",
 968                 "/etc/osm/ro.cfg",
 969             ):
 970                 if path.isfile(config_file):
 971                     break
 972             else:
 973                 print(
 974                     "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
 975                     file=sys.stderr,
 976                 )
 977                 exit(1)
 978
 979         ro_main(config_file)
 980     except KeyboardInterrupt:
 981         print("KeyboardInterrupt. Finishing", file=sys.stderr)
 982     except getopt.GetoptError as e:
 983         print(str(e), file=sys.stderr)
 984         # usage()
 985         exit(1)