osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 import cherrypy
   5 import time
   6 import json
   7 import yaml
   8 import html_out as html
   9 import logging
  10 import logging.handlers
  11 import getopt
  12 import sys
  13
  14 from authconn import AuthException
  15 from auth import Authenticator
  16 from engine import Engine, EngineException
  17 from osm_common.dbbase import DbException
  18 from osm_common.fsbase import FsException
  19 from osm_common.msgbase import MsgException
  20 from http import HTTPStatus
  21 from codecs import getreader
  22 from os import environ, path
  23
  24 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  25
  26 # TODO consider to remove and provide version using the static version file
  27 __version__ = "0.1.3"
  28 version_date = "Apr 2018"
  29 database_version = '1.0'
  30 auth_database_version = '1.0'
  31
  32 """
  33 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  34 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  35         /nsd/v1                                                 O       O
  36             /ns_descriptors_content                             O       O
  37                 /<nsdInfoId>                                    O       O       O       O
  38             /ns_descriptors                                     O5      O5
  39                 /<nsdInfoId>                                    O5                      O5      5
  40                     /nsd_content                                O5              O5
  41                     /nsd                                        O
  42                     /artifacts[/<artifactPath>]                 O
  43             /pnf_descriptors                                    5       5
  44                 /<pnfdInfoId>                                   5                       5       5
  45                     /pnfd_content                               5               5
  46             /subscriptions                                      5       5
  47                 /<subscriptionId>                               5                       X
  48
  49         /vnfpkgm/v1
  50             /vnf_packages_content                               O       O
  51                 /<vnfPkgId>                                     O                       O
  52             /vnf_packages                                       O5      O5
  53                 /<vnfPkgId>                                     O5                      O5      5
  54                     /package_content                            O5               O5
  55                         /upload_from_uri                                X
  56                     /vnfd                                       O5
  57                     /artifacts[/<artifactPath>]                 O5
  58             /subscriptions                                      X       X
  59                 /<subscriptionId>                               X                       X
  60
  61         /nslcm/v1
  62             /ns_instances_content                               O       O
  63                 /<nsInstanceId>                                 O                       O
  64             /ns_instances                                       5       5
  65                 /<nsInstanceId>                                 O5                      O5
  66                     instantiate                                         O5
  67                     terminate                                           O5
  68                     action                                              O
  69                     scale                                               O5
  70                     heal                                                5
  71             /ns_lcm_op_occs                                     5       5
  72                 /<nsLcmOpOccId>                                 5                       5       5
  73                     TO BE COMPLETED                             5               5
  74             /vnf_instances  (also vnfrs for compatibility)      O
  75                 /<vnfInstanceId>                                O
  76             /subscriptions                                      5       5
  77                 /<subscriptionId>                               5                       X
  78         /pdu/v1
  79             /pdu_descriptor                                     O       O
  80                 /<id>                                           O               O       O       O
  81         /admin/v1
  82             /tokens                                             O       O
  83                 /<id>                                           O                       O
  84             /users                                              O       O
  85                 /<id>                                           O               O       O       O
  86             /projects                                           O       O
  87                 /<id>                                           O                       O
  88             /vims_accounts  (also vims for compatibility)       O       O
  89                 /<id>                                           O                       O       O
  90             /sdns                                               O       O
  91                 /<id>                                           O                       O       O
  92
  93 query string:
  94     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
  95     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
  96     item of the array, that is, pass if any item of the array pass the filter.
  97     It allows both ne and neq for not equal
  98     TODO: 4.3.3 Attribute selectors
  99         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 100         (none)  … same as “exclude_default”
 101         all_fields      … all attributes.
 102         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 103         conditionally mandatory, and that are not provided in <list>.
 104         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 105         are not conditionally mandatory, and that are provided in <list>.
 106         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 107         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 108         the particular resource
 109         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 110         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 111         present specification for the particular resource, but that are not part of <list>
 112 Header field name       Reference       Example Descriptions
 113     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 114     This header field shall be present if the response is expected to have a non-empty message body.
 115     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 116     This header field shall be present if the request has a non-empty message body.
 117     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 118     Details are specified in clause 4.5.3.
 119     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 120 Header field name       Reference       Example Descriptions
 121     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 122     This header field shall be present if the response has a non-empty message body.
 123     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 124     new resource has been created.
 125     This header field shall be present if the response status code is 201 or 3xx.
 126     In the present document this header field is also used if the response status code is 202 and a new resource was
 127     created.
 128     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 129     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 130     token.
 131     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 132     certain resources.
 133     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 134     response, and the total length of the file.
 135     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 136 """
 137
 138
 139 class NbiException(Exception):
 140
 141     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 142         Exception.__init__(self, message)
 143         self.http_code = http_code
 144
 145
 146 class Server(object):
 147     instance = 0
 148     # to decode bytes to str
 149     reader = getreader("utf-8")
 150
 151     def __init__(self):
 152         self.instance += 1
 153         self.engine = Engine()
 154         self.authenticator = Authenticator()
 155         self.valid_methods = {   # contains allowed URL and methods
 156             "admin": {
 157                 "v1": {
 158                     "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 159                                "<ID>": {"METHODS": ("GET", "DELETE")}
 160                                },
 161                     "users": {"METHODS": ("GET", "POST"),
 162                               "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT")}
 163                               },
 164                     "projects": {"METHODS": ("GET", "POST"),
 165                                  "<ID>": {"METHODS": ("GET", "DELETE")}
 166                                  },
 167                     "vims": {"METHODS": ("GET", "POST"),
 168                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 169                              },
 170                     "vim_accounts": {"METHODS": ("GET", "POST"),
 171                                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 172                                      },
 173                     "sdns": {"METHODS": ("GET", "POST"),
 174                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 175                              },
 176                 }
 177             },
 178             "pdu": {
 179                 "v1": {
 180                     "pdu_descriptors": {"METHODS": ("GET", "POST"),
 181                                         "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT")}
 182                                         },
 183                 }
 184             },
 185             "nsd": {
 186                 "v1": {
 187                     "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 188                                                "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
 189                                                },
 190                     "ns_descriptors": {"METHODS": ("GET", "POST"),
 191                                        "<ID>": {"METHODS": ("GET", "DELETE"), "TODO": "PATCH",
 192                                                 "nsd_content": {"METHODS": ("GET", "PUT")},
 193                                                 "nsd": {"METHODS": "GET"},  # descriptor inside package
 194                                                 "artifacts": {"*": {"METHODS": "GET"}}
 195                                                 }
 196                                        },
 197                     "pnf_descriptors": {"TODO": ("GET", "POST"),
 198                                         "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 199                                                  "pnfd_content": {"TODO": ("GET", "PUT")}
 200                                                  }
 201                                         },
 202                     "subscriptions": {"TODO": ("GET", "POST"),
 203                                       "<ID>": {"TODO": ("GET", "DELETE")}
 204                                       },
 205                 }
 206             },
 207             "vnfpkgm": {
 208                 "v1": {
 209                     "vnf_packages_content": {"METHODS": ("GET", "POST"),
 210                                              "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
 211                                              },
 212                     "vnf_packages": {"METHODS": ("GET", "POST"),
 213                                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 214                                               "package_content": {"METHODS": ("GET", "PUT"),         # package
 215                                                                   "upload_from_uri": {"TODO": "POST"}
 216                                                                   },
 217                                               "vnfd": {"METHODS": "GET"},                    # descriptor inside package
 218                                               "artifacts": {"*": {"METHODS": "GET"}}
 219                                               }
 220                                      },
 221                     "subscriptions": {"TODO": ("GET", "POST"),
 222                                       "<ID>": {"TODO": ("GET", "DELETE")}
 223                                       },
 224                 }
 225             },
 226             "nslcm": {
 227                 "v1": {
 228                     "ns_instances_content": {"METHODS": ("GET", "POST"),
 229                                              "<ID>": {"METHODS": ("GET", "DELETE")}
 230                                              },
 231                     "ns_instances": {"METHODS": ("GET", "POST"),
 232                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 233                                               "scale": {"METHODS": "POST"},
 234                                               "terminate": {"METHODS": "POST"},
 235                                               "instantiate": {"METHODS": "POST"},
 236                                               "action": {"METHODS": "POST"},
 237                                               }
 238                                      },
 239                     "ns_lcm_op_occs": {"METHODS": "GET",
 240                                        "<ID>": {"METHODS": "GET"},
 241                                        },
 242                     "vnfrs": {"METHODS": ("GET"),
 243                               "<ID>": {"METHODS": ("GET")}
 244                               },
 245                     "vnf_instances": {"METHODS": ("GET"),
 246                                       "<ID>": {"METHODS": ("GET")}
 247                                       },
 248                 }
 249             },
 250         }
 251
 252     def _format_in(self, kwargs):
 253         try:
 254             indata = None
 255             if cherrypy.request.body.length:
 256                 error_text = "Invalid input format "
 257
 258                 if "Content-Type" in cherrypy.request.headers:
 259                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 260                         error_text = "Invalid json format "
 261                         indata = json.load(self.reader(cherrypy.request.body))
 262                         cherrypy.request.headers.pop("Content-File-MD5", None)
 263                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 264                         error_text = "Invalid yaml format "
 265                         indata = yaml.load(cherrypy.request.body)
 266                         cherrypy.request.headers.pop("Content-File-MD5", None)
 267                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 268                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 269                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 270                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 271                         indata = cherrypy.request.body  # .read()
 272                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 273                         if "descriptor_file" in kwargs:
 274                             filecontent = kwargs.pop("descriptor_file")
 275                             if not filecontent.file:
 276                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 277                             indata = filecontent.file  # .read()
 278                             if filecontent.content_type.value:
 279                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 280                     else:
 281                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 282                         #                          "Only 'Content-Type' of type 'application/json' or
 283                         # 'application/yaml' for input format are available")
 284                         error_text = "Invalid yaml format "
 285                         indata = yaml.load(cherrypy.request.body)
 286                         cherrypy.request.headers.pop("Content-File-MD5", None)
 287                 else:
 288                     error_text = "Invalid yaml format "
 289                     indata = yaml.load(cherrypy.request.body)
 290                     cherrypy.request.headers.pop("Content-File-MD5", None)
 291             if not indata:
 292                 indata = {}
 293
 294             format_yaml = False
 295             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 296                 format_yaml = True
 297
 298             for k, v in kwargs.items():
 299                 if isinstance(v, str):
 300                     if v == "":
 301                         kwargs[k] = None
 302                     elif format_yaml:
 303                         try:
 304                             kwargs[k] = yaml.load(v)
 305                         except Exception:
 306                             pass
 307                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 308                         try:
 309                             kwargs[k] = int(v)
 310                         except Exception:
 311                             try:
 312                                 kwargs[k] = float(v)
 313                             except Exception:
 314                                 pass
 315                     elif v.find(",") > 0:
 316                         kwargs[k] = v.split(",")
 317                 elif isinstance(v, (list, tuple)):
 318                     for index in range(0, len(v)):
 319                         if v[index] == "":
 320                             v[index] = None
 321                         elif format_yaml:
 322                             try:
 323                                 v[index] = yaml.load(v[index])
 324                             except Exception:
 325                                 pass
 326
 327             return indata
 328         except (ValueError, yaml.YAMLError) as exc:
 329             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 330         except KeyError as exc:
 331             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 332         except Exception as exc:
 333             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 334
 335     @staticmethod
 336     def _format_out(data, session=None, _format=None):
 337         """
 338         return string of dictionary data according to requested json, yaml, xml. By default json
 339         :param data: response to be sent. Can be a dict, text or file
 340         :param session:
 341         :param _format: The format to be set as Content-Type ir data is a file
 342         :return: None
 343         """
 344         accept = cherrypy.request.headers.get("Accept")
 345         if data is None:
 346             if accept and "text/html" in accept:
 347                 return html.format(data, cherrypy.request, cherrypy.response, session)
 348             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 349             return
 350         elif hasattr(data, "read"):  # file object
 351             if _format:
 352                 cherrypy.response.headers["Content-Type"] = _format
 353             elif "b" in data.mode:  # binariy asssumig zip
 354                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 355             else:
 356                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 357             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 358             return data
 359         if accept:
 360             if "application/json" in accept:
 361                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 362                 a = json.dumps(data, indent=4) + "\n"
 363                 return a.encode("utf8")
 364             elif "text/html" in accept:
 365                 return html.format(data, cherrypy.request, cherrypy.response, session)
 366
 367             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 368                 pass
 369             else:
 370                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 371                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 372                                          "for output format are available")
 373         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 374         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 375                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 376
 377     @cherrypy.expose
 378     def index(self, *args, **kwargs):
 379         session = None
 380         try:
 381             if cherrypy.request.method == "GET":
 382                 session = self.authenticator.authorize()
 383                 outdata = "Index page"
 384             else:
 385                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 386                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 387
 388             return self._format_out(outdata, session)
 389
 390         except (EngineException, AuthException) as e:
 391             cherrypy.log("index Exception {}".format(e))
 392             cherrypy.response.status = e.http_code.value
 393             return self._format_out("Welcome to OSM!", session)
 394
 395     @cherrypy.expose
 396     def version(self, *args, **kwargs):
 397         # TODO consider to remove and provide version using the static version file
 398         global __version__, version_date
 399         try:
 400             if cherrypy.request.method != "GET":
 401                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 402             elif args or kwargs:
 403                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 404             return __version__ + " " + version_date
 405         except NbiException as e:
 406             cherrypy.response.status = e.http_code.value
 407             problem_details = {
 408                 "code": e.http_code.name,
 409                 "status": e.http_code.value,
 410                 "detail": str(e),
 411             }
 412             return self._format_out(problem_details, None)
 413
 414     @cherrypy.expose
 415     def token(self, method, token_id=None, kwargs=None):
 416         session = None
 417         # self.engine.load_dbase(cherrypy.request.app.config)
 418         indata = self._format_in(kwargs)
 419         if not isinstance(indata, dict):
 420             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 421         try:
 422             if method == "GET":
 423                 session = self.authenticator.authorize()
 424                 if token_id:
 425                     outdata = self.authenticator.get_token(session, token_id)
 426                 else:
 427                     outdata = self.authenticator.get_token_list(session)
 428             elif method == "POST":
 429                 try:
 430                     session = self.authenticator.authorize()
 431                 except Exception:
 432                     session = None
 433                 if kwargs:
 434                     indata.update(kwargs)
 435                 outdata = self.authenticator.new_token(session, indata, cherrypy.request.remote)
 436                 session = outdata
 437                 cherrypy.session['Authorization'] = outdata["_id"]
 438                 self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 439                 # cherrypy.response.cookie["Authorization"] = outdata["id"]
 440                 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 441             elif method == "DELETE":
 442                 if not token_id and "id" in kwargs:
 443                     token_id = kwargs["id"]
 444                 elif not token_id:
 445                     session = self.authenticator.authorize()
 446                     token_id = session["_id"]
 447                 outdata = self.authenticator.del_token(token_id)
 448                 session = None
 449                 cherrypy.session['Authorization'] = "logout"
 450                 # cherrypy.response.cookie["Authorization"] = token_id
 451                 # cherrypy.response.cookie["Authorization"]['expires'] = 0
 452             else:
 453                 raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 454             return self._format_out(outdata, session)
 455         except (NbiException, EngineException, DbException, AuthException) as e:
 456             cherrypy.log("tokens Exception {}".format(e))
 457             cherrypy.response.status = e.http_code.value
 458             problem_details = {
 459                 "code": e.http_code.name,
 460                 "status": e.http_code.value,
 461                 "detail": str(e),
 462             }
 463             return self._format_out(problem_details, session)
 464
 465     @cherrypy.expose
 466     def test(self, *args, **kwargs):
 467         thread_info = None
 468         if args and args[0] == "help":
 469             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
 470                    "sleep/<time>\nmessage/topic\n</pre></html>"
 471
 472         elif args and args[0] == "init":
 473             try:
 474                 # self.engine.load_dbase(cherrypy.request.app.config)
 475                 self.engine.create_admin()
 476                 return "Done. User 'admin', password 'admin' created"
 477             except Exception:
 478                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 479                 return self._format_out("Database already initialized")
 480         elif args and args[0] == "file":
 481             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 482                                                   "text/plain", "attachment")
 483         elif args and args[0] == "file2":
 484             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 485             f = open(f_path, "r")
 486             cherrypy.response.headers["Content-type"] = "text/plain"
 487             return f
 488
 489         elif len(args) == 2 and args[0] == "db-clear":
 490             return self.engine.db.del_list(args[1], kwargs)
 491         elif args and args[0] == "prune":
 492             return self.engine.prune()
 493         elif args and args[0] == "login":
 494             if not cherrypy.request.headers.get("Authorization"):
 495                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 496                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 497         elif args and args[0] == "login2":
 498             if not cherrypy.request.headers.get("Authorization"):
 499                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 500                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 501         elif args and args[0] == "sleep":
 502             sleep_time = 5
 503             try:
 504                 sleep_time = int(args[1])
 505             except Exception:
 506                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 507                 return self._format_out("Database already initialized")
 508             thread_info = cherrypy.thread_data
 509             print(thread_info)
 510             time.sleep(sleep_time)
 511             # thread_info
 512         elif len(args) >= 2 and args[0] == "message":
 513             main_topic = args[1]
 514             return_text = "<html><pre>{} ->\n".format(main_topic)
 515             try:
 516                 if cherrypy.request.method == 'POST':
 517                     to_send = yaml.load(cherrypy.request.body)
 518                     for k, v in to_send.items():
 519                         self.engine.msg.write(main_topic, k, v)
 520                         return_text += "  {}: {}\n".format(k, v)
 521                 elif cherrypy.request.method == 'GET':
 522                     for k, v in kwargs.items():
 523                         self.engine.msg.write(main_topic, k, yaml.load(v))
 524                         return_text += "  {}: {}\n".format(k, yaml.load(v))
 525             except Exception as e:
 526                 return_text += "Error: " + str(e)
 527             return_text += "</pre></html>\n"
 528             return return_text
 529
 530         return_text = (
 531             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 532             "  kwargs: {}\n".format(kwargs) +
 533             "  headers: {}\n".format(cherrypy.request.headers) +
 534             "  path_info: {}\n".format(cherrypy.request.path_info) +
 535             "  query_string: {}\n".format(cherrypy.request.query_string) +
 536             "  session: {}\n".format(cherrypy.session) +
 537             "  cookie: {}\n".format(cherrypy.request.cookie) +
 538             "  method: {}\n".format(cherrypy.request.method) +
 539             " session: {}\n".format(cherrypy.session.get('fieldname')) +
 540             "  body:\n")
 541         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 542         if cherrypy.request.body.length:
 543             return_text += "    content: {}\n".format(
 544                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 545         if thread_info:
 546             return_text += "thread: {}\n".format(thread_info)
 547         return_text += "</pre></html>"
 548         return return_text
 549
 550     def _check_valid_url_method(self, method, *args):
 551         if len(args) < 3:
 552             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 553
 554         reference = self.valid_methods
 555         for arg in args:
 556             if arg is None:
 557                 break
 558             if not isinstance(reference, dict):
 559                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 560                                    HTTPStatus.METHOD_NOT_ALLOWED)
 561
 562             if arg in reference:
 563                 reference = reference[arg]
 564             elif "<ID>" in reference:
 565                 reference = reference["<ID>"]
 566             elif "*" in reference:
 567                 reference = reference["*"]
 568                 break
 569             else:
 570                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 571         if "TODO" in reference and method in reference["TODO"]:
 572             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 573         elif "METHODS" in reference and method not in reference["METHODS"]:
 574             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 575         return
 576
 577     @staticmethod
 578     def _set_location_header(main_topic, version, topic, id):
 579         """
 580         Insert response header Location with the URL of created item base on URL params
 581         :param main_topic:
 582         :param version:
 583         :param topic:
 584         :param id:
 585         :return: None
 586         """
 587         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 588         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 589         return
 590
 591     @cherrypy.expose
 592     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 593         session = None
 594         outdata = None
 595         _format = None
 596         method = "DONE"
 597         engine_topic = None
 598         rollback = []
 599         session = None
 600         try:
 601             if not main_topic or not version or not topic:
 602                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 603                                    HTTPStatus.METHOD_NOT_ALLOWED)
 604             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm"):
 605                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 606                                    HTTPStatus.METHOD_NOT_ALLOWED)
 607             if version != 'v1':
 608                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 609
 610             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 611                 method = kwargs.pop("METHOD")
 612             else:
 613                 method = cherrypy.request.method
 614             if kwargs and "FORCE" in kwargs:
 615                 force = kwargs.pop("FORCE")
 616             else:
 617                 force = False
 618
 619             self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 620
 621             if main_topic == "admin" and topic == "tokens":
 622                 return self.token(method, _id, kwargs)
 623
 624             # self.engine.load_dbase(cherrypy.request.app.config)
 625             session = self.authenticator.authorize()
 626             indata = self._format_in(kwargs)
 627             engine_topic = topic
 628             if topic == "subscriptions":
 629                 engine_topic = main_topic + "_" + topic
 630             if item:
 631                 engine_topic = item
 632
 633             if main_topic == "nsd":
 634                 engine_topic = "nsds"
 635             elif main_topic == "vnfpkgm":
 636                 engine_topic = "vnfds"
 637             elif main_topic == "nslcm":
 638                 engine_topic = "nsrs"
 639                 if topic == "ns_lcm_op_occs":
 640                     engine_topic = "nslcmops"
 641                 if topic == "vnfrs" or topic == "vnf_instances":
 642                     engine_topic = "vnfrs"
 643             elif main_topic == "pdu":
 644                 engine_topic = "pdus"
 645             if engine_topic == "vims":   # TODO this is for backward compatibility, it will remove in the future
 646                 engine_topic = "vim_accounts"
 647
 648             if method == "GET":
 649                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd"):
 650                     if item in ("vnfd", "nsd"):
 651                         path = "$DESCRIPTOR"
 652                     elif args:
 653                         path = args
 654                     elif item == "artifacts":
 655                         path = ()
 656                     else:
 657                         path = None
 658                     file, _format = self.engine.get_file(session, engine_topic, _id, path,
 659                                                          cherrypy.request.headers.get("Accept"))
 660                     outdata = file
 661                 elif not _id:
 662                     outdata = self.engine.get_item_list(session, engine_topic, kwargs)
 663                 else:
 664                     outdata = self.engine.get_item(session, engine_topic, _id)
 665             elif method == "POST":
 666                 if topic in ("ns_descriptors_content", "vnf_packages_content"):
 667                     _id = cherrypy.request.headers.get("Transaction-Id")
 668                     if not _id:
 669                         _id = self.engine.new_item(rollback, session, engine_topic, {}, None, cherrypy.request.headers,
 670                                                    force=force)
 671                     completed = self.engine.upload_content(session, engine_topic, _id, indata, kwargs,
 672                                                            cherrypy.request.headers, force=force)
 673                     if completed:
 674                         self._set_location_header(main_topic, version, topic, _id)
 675                     else:
 676                         cherrypy.response.headers["Transaction-Id"] = _id
 677                     outdata = {"id": _id}
 678                 elif topic == "ns_instances_content":
 679                     # creates NSR
 680                     _id = self.engine.new_item(rollback, session, engine_topic, indata, kwargs, force=force)
 681                     # creates nslcmop
 682                     indata["lcmOperationType"] = "instantiate"
 683                     indata["nsInstanceId"] = _id
 684                     self.engine.new_item(rollback, session, "nslcmops", indata, None)
 685                     self._set_location_header(main_topic, version, topic, _id)
 686                     outdata = {"id": _id}
 687                 elif topic == "ns_instances" and item:
 688                     indata["lcmOperationType"] = item
 689                     indata["nsInstanceId"] = _id
 690                     _id = self.engine.new_item(rollback, session, "nslcmops", indata, kwargs)
 691                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
 692                     outdata = {"id": _id}
 693                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
 694                 else:
 695                     _id = self.engine.new_item(rollback, session, engine_topic, indata, kwargs,
 696                                                cherrypy.request.headers, force=force)
 697                     self._set_location_header(main_topic, version, topic, _id)
 698                     outdata = {"id": _id}
 699                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
 700                 cherrypy.response.status = HTTPStatus.CREATED.value
 701
 702             elif method == "DELETE":
 703                 if not _id:
 704                     outdata = self.engine.del_item_list(session, engine_topic, kwargs)
 705                     cherrypy.response.status = HTTPStatus.OK.value
 706                 else:  # len(args) > 1
 707                     if topic == "ns_instances_content" and not force:
 708                         nslcmop_desc = {
 709                             "lcmOperationType": "terminate",
 710                             "nsInstanceId": _id,
 711                             "autoremove": True
 712                         }
 713                         opp_id = self.engine.new_item(rollback, session, "nslcmops", nslcmop_desc, None)
 714                         outdata = {"_id": opp_id}
 715                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
 716                     else:
 717                         self.engine.del_item(session, engine_topic, _id, force)
 718                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 719                 if engine_topic in ("vim_accounts", "sdns"):
 720                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
 721
 722             elif method in ("PUT", "PATCH"):
 723                 outdata = None
 724                 if not indata and not kwargs:
 725                     raise NbiException("Nothing to update. Provide payload and/or query string",
 726                                        HTTPStatus.BAD_REQUEST)
 727                 if item in ("nsd_content", "package_content") and method == "PUT":
 728                     completed = self.engine.upload_content(session, engine_topic, _id, indata, kwargs,
 729                                                            cherrypy.request.headers, force=force)
 730                     if not completed:
 731                         cherrypy.response.headers["Transaction-Id"] = id
 732                 else:
 733                     self.engine.edit_item(session, engine_topic, _id, indata, kwargs, force=force)
 734                 cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 735             else:
 736                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 737             return self._format_out(outdata, session, _format)
 738         except Exception as e:
 739             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException)):
 740                 http_code_value = cherrypy.response.status = e.http_code.value
 741                 http_code_name = e.http_code.name
 742                 cherrypy.log("Exception {}".format(e))
 743             else:
 744                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 745                 cherrypy.log("CRITICAL: Exception {}".format(e))
 746                 http_code_name = HTTPStatus.BAD_REQUEST.name
 747             if hasattr(outdata, "close"):  # is an open file
 748                 outdata.close()
 749             error_text = str(e)
 750             rollback.reverse()
 751             for rollback_item in rollback:
 752                 try:
 753                     self.engine.del_item(**rollback_item, session=session, force=True)
 754                 except Exception as e2:
 755                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
 756                     cherrypy.log(rollback_error_text)
 757                     error_text += ". " + rollback_error_text
 758             # if isinstance(e, MsgException):
 759             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 760             #         engine_topic[:-1], method, error_text)
 761             problem_details = {
 762                 "code": http_code_name,
 763                 "status": http_code_value,
 764                 "detail": error_text,
 765             }
 766             return self._format_out(problem_details, session)
 767             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 768
 769
 770 # def validate_password(realm, username, password):
 771 #     cherrypy.log("realm "+ str(realm))
 772 #     if username == "admin" and password == "admin":
 773 #         return True
 774 #     return False
 775
 776
 777 def _start_service():
 778     """
 779     Callback function called when cherrypy.engine starts
 780     Override configuration with env variables
 781     Set database, storage, message configuration
 782     Init database with admin/admin user password
 783     """
 784     cherrypy.log.error("Starting osm_nbi")
 785     # update general cherrypy configuration
 786     update_dict = {}
 787
 788     engine_config = cherrypy.tree.apps['/osm'].config
 789     for k, v in environ.items():
 790         if not k.startswith("OSMNBI_"):
 791             continue
 792         k1, _, k2 = k[7:].lower().partition("_")
 793         if not k2:
 794             continue
 795         try:
 796             # update static configuration
 797             if k == 'OSMNBI_STATIC_DIR':
 798                 engine_config["/static"]['tools.staticdir.dir'] = v
 799                 engine_config["/static"]['tools.staticdir.on'] = True
 800             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
 801                 update_dict['server.socket_port'] = int(v)
 802             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
 803                 update_dict['server.socket_host'] = v
 804             elif k1 in ("server", "test", "auth", "log"):
 805                 update_dict[k1 + '.' + k2] = v
 806             elif k1 in ("message", "database", "storage", "authentication"):
 807                 # k2 = k2.replace('_', '.')
 808                 if k2 in ("port", "db_port"):
 809                     engine_config[k1][k2] = int(v)
 810                 else:
 811                     engine_config[k1][k2] = v
 812
 813         except ValueError as e:
 814             cherrypy.log.error("Ignoring environ '{}': " + str(e))
 815         except Exception as e:
 816             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
 817
 818     if update_dict:
 819         cherrypy.config.update(update_dict)
 820         engine_config["global"].update(update_dict)
 821
 822     # logging cherrypy
 823     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 824     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 825     logger_server = logging.getLogger("cherrypy.error")
 826     logger_access = logging.getLogger("cherrypy.access")
 827     logger_cherry = logging.getLogger("cherrypy")
 828     logger_nbi = logging.getLogger("nbi")
 829
 830     if "log.file" in engine_config["global"]:
 831         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
 832                                                             maxBytes=100e6, backupCount=9, delay=0)
 833         file_handler.setFormatter(log_formatter_simple)
 834         logger_cherry.addHandler(file_handler)
 835         logger_nbi.addHandler(file_handler)
 836     # log always to standard output
 837     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
 838                             "nbi.access %(filename)s:%(lineno)s": logger_access,
 839                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
 840                             }.items():
 841         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 842         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
 843         str_handler = logging.StreamHandler()
 844         str_handler.setFormatter(log_formatter_cherry)
 845         logger.addHandler(str_handler)
 846
 847     if engine_config["global"].get("log.level"):
 848         logger_cherry.setLevel(engine_config["global"]["log.level"])
 849         logger_nbi.setLevel(engine_config["global"]["log.level"])
 850
 851     # logging other modules
 852     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
 853         engine_config[k1]["logger_name"] = logname
 854         logger_module = logging.getLogger(logname)
 855         if "logfile" in engine_config[k1]:
 856             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
 857                                                                 maxBytes=100e6, backupCount=9, delay=0)
 858             file_handler.setFormatter(log_formatter_simple)
 859             logger_module.addHandler(file_handler)
 860         if "loglevel" in engine_config[k1]:
 861             logger_module.setLevel(engine_config[k1]["loglevel"])
 862     # TODO add more entries, e.g.: storage
 863     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
 864     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
 865     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
 866     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
 867     # getenv('OSMOPENMANO_TENANT', None)
 868
 869
 870 def _stop_service():
 871     """
 872     Callback function called when cherrypy.engine stops
 873     TODO: Ending database connections.
 874     """
 875     cherrypy.tree.apps['/osm'].root.engine.stop()
 876     cherrypy.log.error("Stopping osm_nbi")
 877
 878
 879 def nbi(config_file):
 880     # conf = {
 881     #     '/': {
 882     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
 883     #         'tools.sessions.on': True,
 884     #         'tools.response_headers.on': True,
 885     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
 886     #     }
 887     # }
 888     # cherrypy.Server.ssl_module = 'builtin'
 889     # cherrypy.Server.ssl_certificate = "http/cert.pem"
 890     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
 891     # cherrypy.Server.thread_pool = 10
 892     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
 893
 894     # cherrypy.config.update({'tools.auth_basic.on': True,
 895     #    'tools.auth_basic.realm': 'localhost',
 896     #    'tools.auth_basic.checkpassword': validate_password})
 897     cherrypy.engine.subscribe('start', _start_service)
 898     cherrypy.engine.subscribe('stop', _stop_service)
 899     cherrypy.quickstart(Server(), '/osm', config_file)
 900
 901
 902 def usage():
 903     print("""Usage: {} [options]
 904         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
 905         -h|--help: shows this help
 906         """.format(sys.argv[0]))
 907     # --log-socket-host HOST: send logs to this host")
 908     # --log-socket-port PORT: send logs using this port (default: 9022)")
 909
 910
 911 if __name__ == '__main__':
 912     try:
 913         # load parameters and configuration
 914         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 915         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 916         config_file = None
 917         for o, a in opts:
 918             if o in ("-h", "--help"):
 919                 usage()
 920                 sys.exit()
 921             elif o in ("-c", "--config"):
 922                 config_file = a
 923             # elif o == "--log-socket-port":
 924             #     log_socket_port = a
 925             # elif o == "--log-socket-host":
 926             #     log_socket_host = a
 927             # elif o == "--log-file":
 928             #     log_file = a
 929             else:
 930                 assert False, "Unhandled option"
 931         if config_file:
 932             if not path.isfile(config_file):
 933                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
 934                 exit(1)
 935         else:
 936             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
 937                 if path.isfile(config_file):
 938                     break
 939             else:
 940                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
 941                 exit(1)
 942         nbi(config_file)
 943     except getopt.GetoptError as e:
 944         print(str(e), file=sys.stderr)
 945         # usage()
 946         exit(1)