osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 import cherrypy
   5 import time
   6 import json
   7 import yaml
   8 import html_out as html
   9 import logging
  10 import logging.handlers
  11 import getopt
  12 import sys
  13
  14 from authconn import AuthException
  15 from auth import Authenticator
  16 from engine import Engine, EngineException
  17 from validation import ValidationError
  18 from osm_common.dbbase import DbException
  19 from osm_common.fsbase import FsException
  20 from osm_common.msgbase import MsgException
  21 from http import HTTPStatus
  22 from codecs import getreader
  23 from os import environ, path
  24
  25 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  26
  27 # TODO consider to remove and provide version using the static version file
  28 __version__ = "0.1.3"
  29 version_date = "Apr 2018"
  30 database_version = '1.0'
  31 auth_database_version = '1.0'
  32
  33 """
  34 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  35 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  36         /nsd/v1                                                 O       O
  37             /ns_descriptors_content                             O       O
  38                 /<nsdInfoId>                                    O       O       O       O
  39             /ns_descriptors                                     O5      O5
  40                 /<nsdInfoId>                                    O5                      O5      5
  41                     /nsd_content                                O5              O5
  42                     /nsd                                        O
  43                     /artifacts[/<artifactPath>]                 O
  44             /pnf_descriptors                                    5       5
  45                 /<pnfdInfoId>                                   5                       5       5
  46                     /pnfd_content                               5               5
  47             /subscriptions                                      5       5
  48                 /<subscriptionId>                               5                       X
  49
  50         /vnfpkgm/v1
  51             /vnf_packages_content                               O       O
  52                 /<vnfPkgId>                                     O                       O
  53             /vnf_packages                                       O5      O5
  54                 /<vnfPkgId>                                     O5                      O5      5
  55                     /package_content                            O5               O5
  56                         /upload_from_uri                                X
  57                     /vnfd                                       O5
  58                     /artifacts[/<artifactPath>]                 O5
  59             /subscriptions                                      X       X
  60                 /<subscriptionId>                               X                       X
  61
  62         /nslcm/v1
  63             /ns_instances_content                               O       O
  64                 /<nsInstanceId>                                 O                       O
  65             /ns_instances                                       5       5
  66                 /<nsInstanceId>                                 O5                      O5
  67                     instantiate                                         O5
  68                     terminate                                           O5
  69                     action                                              O
  70                     scale                                               O5
  71                     heal                                                5
  72             /ns_lcm_op_occs                                     5       5
  73                 /<nsLcmOpOccId>                                 5                       5       5
  74                     TO BE COMPLETED                             5               5
  75             /vnf_instances  (also vnfrs for compatibility)      O
  76                 /<vnfInstanceId>                                O
  77             /subscriptions                                      5       5
  78                 /<subscriptionId>                               5                       X
  79         /pdu/v1
  80             /pdu_descriptor                                     O       O
  81                 /<id>                                           O               O       O       O
  82         /admin/v1
  83             /tokens                                             O       O
  84                 /<id>                                           O                       O
  85             /users                                              O       O
  86                 /<id>                                           O               O       O       O
  87             /projects                                           O       O
  88                 /<id>                                           O                       O
  89             /vims_accounts  (also vims for compatibility)       O       O
  90                 /<id>                                           O                       O       O
  91             /sdns                                               O       O
  92                 /<id>                                           O                       O       O
  93
  94 query string:
  95     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
  96         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
  97         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
  98         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
  99         attrName := string
 100     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 101     item of the array, that is, pass if any item of the array pass the filter.
 102     It allows both ne and neq for not equal
 103     TODO: 4.3.3 Attribute selectors
 104         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 105         (none)  … same as “exclude_default”
 106         all_fields      … all attributes.
 107         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 108         conditionally mandatory, and that are not provided in <list>.
 109         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 110         are not conditionally mandatory, and that are provided in <list>.
 111         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 112         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 113         the particular resource
 114         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 115         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 116         present specification for the particular resource, but that are not part of <list>
 117 Header field name       Reference       Example Descriptions
 118     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 119     This header field shall be present if the response is expected to have a non-empty message body.
 120     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 121     This header field shall be present if the request has a non-empty message body.
 122     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 123     Details are specified in clause 4.5.3.
 124     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 125 Header field name       Reference       Example Descriptions
 126     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 127     This header field shall be present if the response has a non-empty message body.
 128     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 129     new resource has been created.
 130     This header field shall be present if the response status code is 201 or 3xx.
 131     In the present document this header field is also used if the response status code is 202 and a new resource was
 132     created.
 133     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 134     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 135     token.
 136     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 137     certain resources.
 138     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 139     response, and the total length of the file.
 140     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 141 """
 142
 143
 144 class NbiException(Exception):
 145
 146     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 147         Exception.__init__(self, message)
 148         self.http_code = http_code
 149
 150
 151 class Server(object):
 152     instance = 0
 153     # to decode bytes to str
 154     reader = getreader("utf-8")
 155
 156     def __init__(self):
 157         self.instance += 1
 158         self.engine = Engine()
 159         self.authenticator = Authenticator()
 160         self.valid_methods = {   # contains allowed URL and methods
 161             "admin": {
 162                 "v1": {
 163                     "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 164                                "<ID>": {"METHODS": ("GET", "DELETE")}
 165                                },
 166                     "users": {"METHODS": ("GET", "POST"),
 167                               "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT")}
 168                               },
 169                     "projects": {"METHODS": ("GET", "POST"),
 170                                  "<ID>": {"METHODS": ("GET", "DELETE")}
 171                                  },
 172                     "vims": {"METHODS": ("GET", "POST"),
 173                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 174                              },
 175                     "vim_accounts": {"METHODS": ("GET", "POST"),
 176                                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 177                                      },
 178                     "sdns": {"METHODS": ("GET", "POST"),
 179                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT")}
 180                              },
 181                 }
 182             },
 183             "pdu": {
 184                 "v1": {
 185                     "pdu_descriptors": {"METHODS": ("GET", "POST"),
 186                                         "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT")}
 187                                         },
 188                 }
 189             },
 190             "nsd": {
 191                 "v1": {
 192                     "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 193                                                "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
 194                                                },
 195                     "ns_descriptors": {"METHODS": ("GET", "POST"),
 196                                        "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 197                                                 "nsd_content": {"METHODS": ("GET", "PUT")},
 198                                                 "nsd": {"METHODS": "GET"},  # descriptor inside package
 199                                                 "artifacts": {"*": {"METHODS": "GET"}}
 200                                                 }
 201                                        },
 202                     "pnf_descriptors": {"TODO": ("GET", "POST"),
 203                                         "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 204                                                  "pnfd_content": {"TODO": ("GET", "PUT")}
 205                                                  }
 206                                         },
 207                     "subscriptions": {"TODO": ("GET", "POST"),
 208                                       "<ID>": {"TODO": ("GET", "DELETE")}
 209                                       },
 210                 }
 211             },
 212             "vnfpkgm": {
 213                 "v1": {
 214                     "vnf_packages_content": {"METHODS": ("GET", "POST"),
 215                                              "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
 216                                              },
 217                     "vnf_packages": {"METHODS": ("GET", "POST"),
 218                                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 219                                               "package_content": {"METHODS": ("GET", "PUT"),         # package
 220                                                                   "upload_from_uri": {"TODO": "POST"}
 221                                                                   },
 222                                               "vnfd": {"METHODS": "GET"},                    # descriptor inside package
 223                                               "artifacts": {"*": {"METHODS": "GET"}}
 224                                               }
 225                                      },
 226                     "subscriptions": {"TODO": ("GET", "POST"),
 227                                       "<ID>": {"TODO": ("GET", "DELETE")}
 228                                       },
 229                 }
 230             },
 231             "nslcm": {
 232                 "v1": {
 233                     "ns_instances_content": {"METHODS": ("GET", "POST"),
 234                                              "<ID>": {"METHODS": ("GET", "DELETE")}
 235                                              },
 236                     "ns_instances": {"METHODS": ("GET", "POST"),
 237                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 238                                               "scale": {"METHODS": "POST"},
 239                                               "terminate": {"METHODS": "POST"},
 240                                               "instantiate": {"METHODS": "POST"},
 241                                               "action": {"METHODS": "POST"},
 242                                               }
 243                                      },
 244                     "ns_lcm_op_occs": {"METHODS": "GET",
 245                                        "<ID>": {"METHODS": "GET"},
 246                                        },
 247                     "vnfrs": {"METHODS": ("GET"),
 248                               "<ID>": {"METHODS": ("GET")}
 249                               },
 250                     "vnf_instances": {"METHODS": ("GET"),
 251                                       "<ID>": {"METHODS": ("GET")}
 252                                       },
 253                 }
 254             },
 255         }
 256
 257     def _format_in(self, kwargs):
 258         try:
 259             indata = None
 260             if cherrypy.request.body.length:
 261                 error_text = "Invalid input format "
 262
 263                 if "Content-Type" in cherrypy.request.headers:
 264                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 265                         error_text = "Invalid json format "
 266                         indata = json.load(self.reader(cherrypy.request.body))
 267                         cherrypy.request.headers.pop("Content-File-MD5", None)
 268                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 269                         error_text = "Invalid yaml format "
 270                         indata = yaml.load(cherrypy.request.body)
 271                         cherrypy.request.headers.pop("Content-File-MD5", None)
 272                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 273                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 274                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 275                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 276                         indata = cherrypy.request.body  # .read()
 277                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 278                         if "descriptor_file" in kwargs:
 279                             filecontent = kwargs.pop("descriptor_file")
 280                             if not filecontent.file:
 281                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 282                             indata = filecontent.file  # .read()
 283                             if filecontent.content_type.value:
 284                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 285                     else:
 286                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 287                         #                          "Only 'Content-Type' of type 'application/json' or
 288                         # 'application/yaml' for input format are available")
 289                         error_text = "Invalid yaml format "
 290                         indata = yaml.load(cherrypy.request.body)
 291                         cherrypy.request.headers.pop("Content-File-MD5", None)
 292                 else:
 293                     error_text = "Invalid yaml format "
 294                     indata = yaml.load(cherrypy.request.body)
 295                     cherrypy.request.headers.pop("Content-File-MD5", None)
 296             if not indata:
 297                 indata = {}
 298
 299             format_yaml = False
 300             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 301                 format_yaml = True
 302
 303             for k, v in kwargs.items():
 304                 if isinstance(v, str):
 305                     if v == "":
 306                         kwargs[k] = None
 307                     elif format_yaml:
 308                         try:
 309                             kwargs[k] = yaml.load(v)
 310                         except Exception:
 311                             pass
 312                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 313                         try:
 314                             kwargs[k] = int(v)
 315                         except Exception:
 316                             try:
 317                                 kwargs[k] = float(v)
 318                             except Exception:
 319                                 pass
 320                     elif v.find(",") > 0:
 321                         kwargs[k] = v.split(",")
 322                 elif isinstance(v, (list, tuple)):
 323                     for index in range(0, len(v)):
 324                         if v[index] == "":
 325                             v[index] = None
 326                         elif format_yaml:
 327                             try:
 328                                 v[index] = yaml.load(v[index])
 329                             except Exception:
 330                                 pass
 331
 332             return indata
 333         except (ValueError, yaml.YAMLError) as exc:
 334             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 335         except KeyError as exc:
 336             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 337         except Exception as exc:
 338             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 339
 340     @staticmethod
 341     def _format_out(data, session=None, _format=None):
 342         """
 343         return string of dictionary data according to requested json, yaml, xml. By default json
 344         :param data: response to be sent. Can be a dict, text or file
 345         :param session:
 346         :param _format: The format to be set as Content-Type ir data is a file
 347         :return: None
 348         """
 349         accept = cherrypy.request.headers.get("Accept")
 350         if data is None:
 351             if accept and "text/html" in accept:
 352                 return html.format(data, cherrypy.request, cherrypy.response, session)
 353             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 354             return
 355         elif hasattr(data, "read"):  # file object
 356             if _format:
 357                 cherrypy.response.headers["Content-Type"] = _format
 358             elif "b" in data.mode:  # binariy asssumig zip
 359                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 360             else:
 361                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 362             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 363             return data
 364         if accept:
 365             if "application/json" in accept:
 366                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 367                 a = json.dumps(data, indent=4) + "\n"
 368                 return a.encode("utf8")
 369             elif "text/html" in accept:
 370                 return html.format(data, cherrypy.request, cherrypy.response, session)
 371
 372             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 373                 pass
 374             else:
 375                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 376                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 377                                          "for output format are available")
 378         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 379         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 380                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 381
 382     @cherrypy.expose
 383     def index(self, *args, **kwargs):
 384         session = None
 385         try:
 386             if cherrypy.request.method == "GET":
 387                 session = self.authenticator.authorize()
 388                 outdata = "Index page"
 389             else:
 390                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 391                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 392
 393             return self._format_out(outdata, session)
 394
 395         except (EngineException, AuthException) as e:
 396             cherrypy.log("index Exception {}".format(e))
 397             cherrypy.response.status = e.http_code.value
 398             return self._format_out("Welcome to OSM!", session)
 399
 400     @cherrypy.expose
 401     def version(self, *args, **kwargs):
 402         # TODO consider to remove and provide version using the static version file
 403         global __version__, version_date
 404         try:
 405             if cherrypy.request.method != "GET":
 406                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 407             elif args or kwargs:
 408                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 409             return __version__ + " " + version_date
 410         except NbiException as e:
 411             cherrypy.response.status = e.http_code.value
 412             problem_details = {
 413                 "code": e.http_code.name,
 414                 "status": e.http_code.value,
 415                 "detail": str(e),
 416             }
 417             return self._format_out(problem_details, None)
 418
 419     @cherrypy.expose
 420     def token(self, method, token_id=None, kwargs=None):
 421         session = None
 422         # self.engine.load_dbase(cherrypy.request.app.config)
 423         indata = self._format_in(kwargs)
 424         if not isinstance(indata, dict):
 425             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 426         try:
 427             if method == "GET":
 428                 session = self.authenticator.authorize()
 429                 if token_id:
 430                     outdata = self.authenticator.get_token(session, token_id)
 431                 else:
 432                     outdata = self.authenticator.get_token_list(session)
 433             elif method == "POST":
 434                 try:
 435                     session = self.authenticator.authorize()
 436                 except Exception:
 437                     session = None
 438                 if kwargs:
 439                     indata.update(kwargs)
 440                 outdata = self.authenticator.new_token(session, indata, cherrypy.request.remote)
 441                 session = outdata
 442                 cherrypy.session['Authorization'] = outdata["_id"]
 443                 self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 444                 # cherrypy.response.cookie["Authorization"] = outdata["id"]
 445                 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 446             elif method == "DELETE":
 447                 if not token_id and "id" in kwargs:
 448                     token_id = kwargs["id"]
 449                 elif not token_id:
 450                     session = self.authenticator.authorize()
 451                     token_id = session["_id"]
 452                 outdata = self.authenticator.del_token(token_id)
 453                 session = None
 454                 cherrypy.session['Authorization'] = "logout"
 455                 # cherrypy.response.cookie["Authorization"] = token_id
 456                 # cherrypy.response.cookie["Authorization"]['expires'] = 0
 457             else:
 458                 raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 459             return self._format_out(outdata, session)
 460         except (NbiException, EngineException, DbException, AuthException) as e:
 461             cherrypy.log("tokens Exception {}".format(e))
 462             cherrypy.response.status = e.http_code.value
 463             problem_details = {
 464                 "code": e.http_code.name,
 465                 "status": e.http_code.value,
 466                 "detail": str(e),
 467             }
 468             return self._format_out(problem_details, session)
 469
 470     @cherrypy.expose
 471     def test(self, *args, **kwargs):
 472         thread_info = None
 473         if args and args[0] == "help":
 474             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
 475                    "sleep/<time>\nmessage/topic\n</pre></html>"
 476
 477         elif args and args[0] == "init":
 478             try:
 479                 # self.engine.load_dbase(cherrypy.request.app.config)
 480                 self.engine.create_admin()
 481                 return "Done. User 'admin', password 'admin' created"
 482             except Exception:
 483                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 484                 return self._format_out("Database already initialized")
 485         elif args and args[0] == "file":
 486             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 487                                                   "text/plain", "attachment")
 488         elif args and args[0] == "file2":
 489             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 490             f = open(f_path, "r")
 491             cherrypy.response.headers["Content-type"] = "text/plain"
 492             return f
 493
 494         elif len(args) == 2 and args[0] == "db-clear":
 495             return self.engine.db.del_list(args[1], kwargs)
 496         elif args and args[0] == "prune":
 497             return self.engine.prune()
 498         elif args and args[0] == "login":
 499             if not cherrypy.request.headers.get("Authorization"):
 500                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 501                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 502         elif args and args[0] == "login2":
 503             if not cherrypy.request.headers.get("Authorization"):
 504                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 505                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 506         elif args and args[0] == "sleep":
 507             sleep_time = 5
 508             try:
 509                 sleep_time = int(args[1])
 510             except Exception:
 511                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 512                 return self._format_out("Database already initialized")
 513             thread_info = cherrypy.thread_data
 514             print(thread_info)
 515             time.sleep(sleep_time)
 516             # thread_info
 517         elif len(args) >= 2 and args[0] == "message":
 518             main_topic = args[1]
 519             return_text = "<html><pre>{} ->\n".format(main_topic)
 520             try:
 521                 if cherrypy.request.method == 'POST':
 522                     to_send = yaml.load(cherrypy.request.body)
 523                     for k, v in to_send.items():
 524                         self.engine.msg.write(main_topic, k, v)
 525                         return_text += "  {}: {}\n".format(k, v)
 526                 elif cherrypy.request.method == 'GET':
 527                     for k, v in kwargs.items():
 528                         self.engine.msg.write(main_topic, k, yaml.load(v))
 529                         return_text += "  {}: {}\n".format(k, yaml.load(v))
 530             except Exception as e:
 531                 return_text += "Error: " + str(e)
 532             return_text += "</pre></html>\n"
 533             return return_text
 534
 535         return_text = (
 536             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 537             "  kwargs: {}\n".format(kwargs) +
 538             "  headers: {}\n".format(cherrypy.request.headers) +
 539             "  path_info: {}\n".format(cherrypy.request.path_info) +
 540             "  query_string: {}\n".format(cherrypy.request.query_string) +
 541             "  session: {}\n".format(cherrypy.session) +
 542             "  cookie: {}\n".format(cherrypy.request.cookie) +
 543             "  method: {}\n".format(cherrypy.request.method) +
 544             " session: {}\n".format(cherrypy.session.get('fieldname')) +
 545             "  body:\n")
 546         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 547         if cherrypy.request.body.length:
 548             return_text += "    content: {}\n".format(
 549                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 550         if thread_info:
 551             return_text += "thread: {}\n".format(thread_info)
 552         return_text += "</pre></html>"
 553         return return_text
 554
 555     def _check_valid_url_method(self, method, *args):
 556         if len(args) < 3:
 557             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 558
 559         reference = self.valid_methods
 560         for arg in args:
 561             if arg is None:
 562                 break
 563             if not isinstance(reference, dict):
 564                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 565                                    HTTPStatus.METHOD_NOT_ALLOWED)
 566
 567             if arg in reference:
 568                 reference = reference[arg]
 569             elif "<ID>" in reference:
 570                 reference = reference["<ID>"]
 571             elif "*" in reference:
 572                 reference = reference["*"]
 573                 break
 574             else:
 575                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 576         if "TODO" in reference and method in reference["TODO"]:
 577             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 578         elif "METHODS" in reference and method not in reference["METHODS"]:
 579             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 580         return
 581
 582     @staticmethod
 583     def _set_location_header(main_topic, version, topic, id):
 584         """
 585         Insert response header Location with the URL of created item base on URL params
 586         :param main_topic:
 587         :param version:
 588         :param topic:
 589         :param id:
 590         :return: None
 591         """
 592         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 593         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 594         return
 595
 596     @cherrypy.expose
 597     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 598         session = None
 599         outdata = None
 600         _format = None
 601         method = "DONE"
 602         engine_topic = None
 603         rollback = []
 604         session = None
 605         try:
 606             if not main_topic or not version or not topic:
 607                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 608                                    HTTPStatus.METHOD_NOT_ALLOWED)
 609             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu"):
 610                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 611                                    HTTPStatus.METHOD_NOT_ALLOWED)
 612             if version != 'v1':
 613                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 614
 615             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 616                 method = kwargs.pop("METHOD")
 617             else:
 618                 method = cherrypy.request.method
 619             if kwargs and "FORCE" in kwargs:
 620                 force = kwargs.pop("FORCE")
 621             else:
 622                 force = False
 623
 624             self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 625
 626             if main_topic == "admin" and topic == "tokens":
 627                 return self.token(method, _id, kwargs)
 628
 629             # self.engine.load_dbase(cherrypy.request.app.config)
 630             session = self.authenticator.authorize()
 631             indata = self._format_in(kwargs)
 632             engine_topic = topic
 633             if topic == "subscriptions":
 634                 engine_topic = main_topic + "_" + topic
 635             if item:
 636                 engine_topic = item
 637
 638             if main_topic == "nsd":
 639                 engine_topic = "nsds"
 640             elif main_topic == "vnfpkgm":
 641                 engine_topic = "vnfds"
 642             elif main_topic == "nslcm":
 643                 engine_topic = "nsrs"
 644                 if topic == "ns_lcm_op_occs":
 645                     engine_topic = "nslcmops"
 646                 if topic == "vnfrs" or topic == "vnf_instances":
 647                     engine_topic = "vnfrs"
 648             elif main_topic == "pdu":
 649                 engine_topic = "pdus"
 650             if engine_topic == "vims":   # TODO this is for backward compatibility, it will remove in the future
 651                 engine_topic = "vim_accounts"
 652
 653             if method == "GET":
 654                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd"):
 655                     if item in ("vnfd", "nsd"):
 656                         path = "$DESCRIPTOR"
 657                     elif args:
 658                         path = args
 659                     elif item == "artifacts":
 660                         path = ()
 661                     else:
 662                         path = None
 663                     file, _format = self.engine.get_file(session, engine_topic, _id, path,
 664                                                          cherrypy.request.headers.get("Accept"))
 665                     outdata = file
 666                 elif not _id:
 667                     outdata = self.engine.get_item_list(session, engine_topic, kwargs)
 668                 else:
 669                     outdata = self.engine.get_item(session, engine_topic, _id)
 670             elif method == "POST":
 671                 if topic in ("ns_descriptors_content", "vnf_packages_content"):
 672                     _id = cherrypy.request.headers.get("Transaction-Id")
 673                     if not _id:
 674                         _id = self.engine.new_item(rollback, session, engine_topic, {}, None, cherrypy.request.headers,
 675                                                    force=force)
 676                     completed = self.engine.upload_content(session, engine_topic, _id, indata, kwargs,
 677                                                            cherrypy.request.headers, force=force)
 678                     if completed:
 679                         self._set_location_header(main_topic, version, topic, _id)
 680                     else:
 681                         cherrypy.response.headers["Transaction-Id"] = _id
 682                     outdata = {"id": _id}
 683                 elif topic == "ns_instances_content":
 684                     # creates NSR
 685                     _id = self.engine.new_item(rollback, session, engine_topic, indata, kwargs, force=force)
 686                     # creates nslcmop
 687                     indata["lcmOperationType"] = "instantiate"
 688                     indata["nsInstanceId"] = _id
 689                     self.engine.new_item(rollback, session, "nslcmops", indata, None)
 690                     self._set_location_header(main_topic, version, topic, _id)
 691                     outdata = {"id": _id}
 692                 elif topic == "ns_instances" and item:
 693                     indata["lcmOperationType"] = item
 694                     indata["nsInstanceId"] = _id
 695                     _id = self.engine.new_item(rollback, session, "nslcmops", indata, kwargs)
 696                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
 697                     outdata = {"id": _id}
 698                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
 699                 else:
 700                     _id = self.engine.new_item(rollback, session, engine_topic, indata, kwargs,
 701                                                cherrypy.request.headers, force=force)
 702                     self._set_location_header(main_topic, version, topic, _id)
 703                     outdata = {"id": _id}
 704                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
 705                 cherrypy.response.status = HTTPStatus.CREATED.value
 706
 707             elif method == "DELETE":
 708                 if not _id:
 709                     outdata = self.engine.del_item_list(session, engine_topic, kwargs)
 710                     cherrypy.response.status = HTTPStatus.OK.value
 711                 else:  # len(args) > 1
 712                     if topic == "ns_instances_content" and not force:
 713                         nslcmop_desc = {
 714                             "lcmOperationType": "terminate",
 715                             "nsInstanceId": _id,
 716                             "autoremove": True
 717                         }
 718                         opp_id = self.engine.new_item(rollback, session, "nslcmops", nslcmop_desc, None)
 719                         outdata = {"_id": opp_id}
 720                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
 721                     else:
 722                         self.engine.del_item(session, engine_topic, _id, force)
 723                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 724                 if engine_topic in ("vim_accounts", "sdns"):
 725                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
 726
 727             elif method in ("PUT", "PATCH"):
 728                 outdata = None
 729                 if not indata and not kwargs:
 730                     raise NbiException("Nothing to update. Provide payload and/or query string",
 731                                        HTTPStatus.BAD_REQUEST)
 732                 if item in ("nsd_content", "package_content") and method == "PUT":
 733                     completed = self.engine.upload_content(session, engine_topic, _id, indata, kwargs,
 734                                                            cherrypy.request.headers, force=force)
 735                     if not completed:
 736                         cherrypy.response.headers["Transaction-Id"] = id
 737                 else:
 738                     self.engine.edit_item(session, engine_topic, _id, indata, kwargs, force=force)
 739                 cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 740             else:
 741                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 742             return self._format_out(outdata, session, _format)
 743         except Exception as e:
 744             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
 745                               ValidationError)):
 746                 http_code_value = cherrypy.response.status = e.http_code.value
 747                 http_code_name = e.http_code.name
 748                 cherrypy.log("Exception {}".format(e))
 749             else:
 750                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
 751                 cherrypy.log("CRITICAL: Exception {}".format(e))
 752                 http_code_name = HTTPStatus.BAD_REQUEST.name
 753             if hasattr(outdata, "close"):  # is an open file
 754                 outdata.close()
 755             error_text = str(e)
 756             rollback.reverse()
 757             for rollback_item in rollback:
 758                 try:
 759                     if rollback_item.get("operation") == "set":
 760                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
 761                                                rollback_item["content"], fail_on_empty=False)
 762                     else:
 763                         self.engine.del_item(**rollback_item, session=session, force=True)
 764                 except Exception as e2:
 765                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
 766                     cherrypy.log(rollback_error_text)
 767                     error_text += ". " + rollback_error_text
 768             # if isinstance(e, MsgException):
 769             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
 770             #         engine_topic[:-1], method, error_text)
 771             problem_details = {
 772                 "code": http_code_name,
 773                 "status": http_code_value,
 774                 "detail": error_text,
 775             }
 776             return self._format_out(problem_details, session)
 777             # raise cherrypy.HTTPError(e.http_code.value, str(e))
 778
 779
 780 # def validate_password(realm, username, password):
 781 #     cherrypy.log("realm "+ str(realm))
 782 #     if username == "admin" and password == "admin":
 783 #         return True
 784 #     return False
 785
 786
 787 def _start_service():
 788     """
 789     Callback function called when cherrypy.engine starts
 790     Override configuration with env variables
 791     Set database, storage, message configuration
 792     Init database with admin/admin user password
 793     """
 794     cherrypy.log.error("Starting osm_nbi")
 795     # update general cherrypy configuration
 796     update_dict = {}
 797
 798     engine_config = cherrypy.tree.apps['/osm'].config
 799     for k, v in environ.items():
 800         if not k.startswith("OSMNBI_"):
 801             continue
 802         k1, _, k2 = k[7:].lower().partition("_")
 803         if not k2:
 804             continue
 805         try:
 806             # update static configuration
 807             if k == 'OSMNBI_STATIC_DIR':
 808                 engine_config["/static"]['tools.staticdir.dir'] = v
 809                 engine_config["/static"]['tools.staticdir.on'] = True
 810             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
 811                 update_dict['server.socket_port'] = int(v)
 812             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
 813                 update_dict['server.socket_host'] = v
 814             elif k1 in ("server", "test", "auth", "log"):
 815                 update_dict[k1 + '.' + k2] = v
 816             elif k1 in ("message", "database", "storage", "authentication"):
 817                 # k2 = k2.replace('_', '.')
 818                 if k2 in ("port", "db_port"):
 819                     engine_config[k1][k2] = int(v)
 820                 else:
 821                     engine_config[k1][k2] = v
 822
 823         except ValueError as e:
 824             cherrypy.log.error("Ignoring environ '{}': " + str(e))
 825         except Exception as e:
 826             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
 827
 828     if update_dict:
 829         cherrypy.config.update(update_dict)
 830         engine_config["global"].update(update_dict)
 831
 832     # logging cherrypy
 833     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
 834     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
 835     logger_server = logging.getLogger("cherrypy.error")
 836     logger_access = logging.getLogger("cherrypy.access")
 837     logger_cherry = logging.getLogger("cherrypy")
 838     logger_nbi = logging.getLogger("nbi")
 839
 840     if "log.file" in engine_config["global"]:
 841         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
 842                                                             maxBytes=100e6, backupCount=9, delay=0)
 843         file_handler.setFormatter(log_formatter_simple)
 844         logger_cherry.addHandler(file_handler)
 845         logger_nbi.addHandler(file_handler)
 846     # log always to standard output
 847     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
 848                             "nbi.access %(filename)s:%(lineno)s": logger_access,
 849                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
 850                             }.items():
 851         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
 852         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
 853         str_handler = logging.StreamHandler()
 854         str_handler.setFormatter(log_formatter_cherry)
 855         logger.addHandler(str_handler)
 856
 857     if engine_config["global"].get("log.level"):
 858         logger_cherry.setLevel(engine_config["global"]["log.level"])
 859         logger_nbi.setLevel(engine_config["global"]["log.level"])
 860
 861     # logging other modules
 862     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
 863         engine_config[k1]["logger_name"] = logname
 864         logger_module = logging.getLogger(logname)
 865         if "logfile" in engine_config[k1]:
 866             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
 867                                                                 maxBytes=100e6, backupCount=9, delay=0)
 868             file_handler.setFormatter(log_formatter_simple)
 869             logger_module.addHandler(file_handler)
 870         if "loglevel" in engine_config[k1]:
 871             logger_module.setLevel(engine_config[k1]["loglevel"])
 872     # TODO add more entries, e.g.: storage
 873     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
 874     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
 875     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
 876     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
 877     # getenv('OSMOPENMANO_TENANT', None)
 878
 879
 880 def _stop_service():
 881     """
 882     Callback function called when cherrypy.engine stops
 883     TODO: Ending database connections.
 884     """
 885     cherrypy.tree.apps['/osm'].root.engine.stop()
 886     cherrypy.log.error("Stopping osm_nbi")
 887
 888
 889 def nbi(config_file):
 890     # conf = {
 891     #     '/': {
 892     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
 893     #         'tools.sessions.on': True,
 894     #         'tools.response_headers.on': True,
 895     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
 896     #     }
 897     # }
 898     # cherrypy.Server.ssl_module = 'builtin'
 899     # cherrypy.Server.ssl_certificate = "http/cert.pem"
 900     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
 901     # cherrypy.Server.thread_pool = 10
 902     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
 903
 904     # cherrypy.config.update({'tools.auth_basic.on': True,
 905     #    'tools.auth_basic.realm': 'localhost',
 906     #    'tools.auth_basic.checkpassword': validate_password})
 907     cherrypy.engine.subscribe('start', _start_service)
 908     cherrypy.engine.subscribe('stop', _stop_service)
 909     cherrypy.quickstart(Server(), '/osm', config_file)
 910
 911
 912 def usage():
 913     print("""Usage: {} [options]
 914         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
 915         -h|--help: shows this help
 916         """.format(sys.argv[0]))
 917     # --log-socket-host HOST: send logs to this host")
 918     # --log-socket-port PORT: send logs using this port (default: 9022)")
 919
 920
 921 if __name__ == '__main__':
 922     try:
 923         # load parameters and configuration
 924         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
 925         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
 926         config_file = None
 927         for o, a in opts:
 928             if o in ("-h", "--help"):
 929                 usage()
 930                 sys.exit()
 931             elif o in ("-c", "--config"):
 932                 config_file = a
 933             # elif o == "--log-socket-port":
 934             #     log_socket_port = a
 935             # elif o == "--log-socket-host":
 936             #     log_socket_host = a
 937             # elif o == "--log-file":
 938             #     log_file = a
 939             else:
 940                 assert False, "Unhandled option"
 941         if config_file:
 942             if not path.isfile(config_file):
 943                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
 944                 exit(1)
 945         else:
 946             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
 947                 if path.isfile(config_file):
 948                     break
 949             else:
 950                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
 951                 exit(1)
 952         nbi(config_file)
 953     except getopt.GetoptError as e:
 954         print(str(e), file=sys.stderr)
 955         # usage()
 956         exit(1)