osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118
 119         /nst/v1                                                 O       O
 120             /netslice_templates_content                         O       O
 121                 /<nstInfoId>                                    O       O       O       O
 122             /netslice_templates                                 O       O
 123                 /<nstInfoId>                                    O                       O       O
 124                     /nst_content                                O               O
 125                     /nst                                        O
 126                     /artifacts[/<artifactPath>]                 O
 127             /subscriptions                                      X       X
 128                 /<subscriptionId>                               X                       X
 129
 130         /nsilcm/v1
 131             /netslice_instances_content                         O       O
 132                 /<SliceInstanceId>                              O                       O
 133             /netslice_instances                                 O       O
 134                 /<SliceInstanceId>                              O                       O
 135                     instantiate                                         O
 136                     terminate                                           O
 137                     action                                              O
 138             /nsi_lcm_op_occs                                    O       O
 139                 /<nsiLcmOpOccId>                                O                       O       O
 140             /subscriptions                                      X       X
 141                 /<subscriptionId>                               X                       X
 142
 143 query string:
 144     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 145         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 146         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 147         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 148         attrName := string
 149     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 150     item of the array, that is, pass if any item of the array pass the filter.
 151     It allows both ne and neq for not equal
 152     TODO: 4.3.3 Attribute selectors
 153         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 154         (none)  … same as “exclude_default”
 155         all_fields      … all attributes.
 156         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 157         conditionally mandatory, and that are not provided in <list>.
 158         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 159         are not conditionally mandatory, and that are provided in <list>.
 160         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 161         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 162         the particular resource
 163         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 164         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 165         present specification for the particular resource, but that are not part of <list>
 166     Additionally it admits some administrator values:
 167         FORCE: To force operations skipping dependency checkings
 168         ADMIN: To act as an administrator or a different project
 169         PUBLIC: To get public descriptors or set a descriptor as public
 170         SET_PROJECT: To make a descriptor available for other project
 171
 172 Header field name       Reference       Example Descriptions
 173     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 174     This header field shall be present if the response is expected to have a non-empty message body.
 175     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 176     This header field shall be present if the request has a non-empty message body.
 177     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 178     Details are specified in clause 4.5.3.
 179     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 180 Header field name       Reference       Example Descriptions
 181     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 182     This header field shall be present if the response has a non-empty message body.
 183     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 184     new resource has been created.
 185     This header field shall be present if the response status code is 201 or 3xx.
 186     In the present document this header field is also used if the response status code is 202 and a new resource was
 187     created.
 188     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 189     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 190     token.
 191     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 192     certain resources.
 193     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 194     response, and the total length of the file.
 195     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 196 """
 197
 198 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 199 # ^ Contains possible administrative query string words:
 200 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 201 #           (not owned by my session project).
 202 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 203 #     FORCE=True(by default)|False: Force edition/deletion operations
 204 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 205
 206 valid_url_methods = {
 207     # contains allowed URL and methods, and the role_permission name
 208     "admin": {
 209         "v1": {
 210             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 211                        "ROLE_PERMISSION": "tokens:",
 212                        "<ID>": {"METHODS": ("GET", "DELETE"),
 213                                 "ROLE_PERMISSION": "tokens:id:"
 214                                 }
 215                        },
 216             "users": {"METHODS": ("GET", "POST"),
 217                       "ROLE_PERMISSION": "users:",
 218                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 219                                "ROLE_PERMISSION": "users:id:"
 220                                }
 221                       },
 222             "projects": {"METHODS": ("GET", "POST"),
 223                          "ROLE_PERMISSION": "projects:",
 224                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 225                                   "ROLE_PERMISSION": "projects:id:"}
 226                          },
 227             "roles": {"METHODS": ("GET", "POST"),
 228                       "ROLE_PERMISSION": "roles:",
 229                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 230                                "ROLE_PERMISSION": "roles:id:"
 231                                }
 232                       },
 233             "vims": {"METHODS": ("GET", "POST"),
 234                      "ROLE_PERMISSION": "vims:",
 235                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 236                               "ROLE_PERMISSION": "vims:id:"
 237                               }
 238                      },
 239             "vim_accounts": {"METHODS": ("GET", "POST"),
 240                              "ROLE_PERMISSION": "vim_accounts:",
 241                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 242                                       "ROLE_PERMISSION": "vim_accounts:id:"
 243                                       }
 244                              },
 245             "wim_accounts": {"METHODS": ("GET", "POST"),
 246                              "ROLE_PERMISSION": "wim_accounts:",
 247                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 248                                       "ROLE_PERMISSION": "wim_accounts:id:"
 249                                       }
 250                              },
 251             "sdns": {"METHODS": ("GET", "POST"),
 252                      "ROLE_PERMISSION": "sdn_controllers:",
 253                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 254                               "ROLE_PERMISSION": "sdn_controllers:id:"
 255                               }
 256                      },
 257             "k8sclusters": {"METHODS": ("GET", "POST"),
 258                             "ROLE_PERMISSION": "k8sclusters:",
 259                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 260                                      "ROLE_PERMISSION": "k8sclusters:id:"
 261                                      }
 262                             },
 263             "k8srepos": {"METHODS": ("GET", "POST"),
 264                          "ROLE_PERMISSION": "k8srepos:",
 265                          "<ID>": {"METHODS": ("GET", "DELETE"),
 266                                   "ROLE_PERMISSION": "k8srepos:id:"
 267                                   }
 268                          },
 269             "domains": {"METHODS": ("GET", ),
 270                         "ROLE_PERMISSION": "domains:",
 271                         },
 272         }
 273     },
 274     "pdu": {
 275         "v1": {
 276             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 277                                 "ROLE_PERMISSION": "pduds:",
 278                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 279                                          "ROLE_PERMISSION": "pduds:id:"
 280                                          }
 281                                 },
 282         }
 283     },
 284     "nsd": {
 285         "v1": {
 286             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 287                                        "ROLE_PERMISSION": "nsds:",
 288                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 289                                                 "ROLE_PERMISSION": "nsds:id:"
 290                                                 }
 291                                        },
 292             "ns_descriptors": {"METHODS": ("GET", "POST"),
 293                                "ROLE_PERMISSION": "nsds:",
 294                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 295                                         "ROLE_PERMISSION": "nsds:id:",
 296                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 297                                                         "ROLE_PERMISSION": "nsds:id:content:",
 298                                                         },
 299                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 300                                                 "ROLE_PERMISSION": "nsds:id:content:"
 301                                                 },
 302                                         "artifacts": {"*": {"METHODS": ("GET",),
 303                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 304                                                             }
 305                                                       }
 306                                         }
 307                                },
 308             "pnf_descriptors": {"TODO": ("GET", "POST"),
 309                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 310                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 311                                          }
 312                                 },
 313             "subscriptions": {"TODO": ("GET", "POST"),
 314                               "<ID>": {"TODO": ("GET", "DELETE")}
 315                               },
 316         }
 317     },
 318     "vnfpkgm": {
 319         "v1": {
 320             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 321                                      "ROLE_PERMISSION": "vnfds:",
 322                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 323                                               "ROLE_PERMISSION": "vnfds:id:"}
 324                                      },
 325             "vnf_packages": {"METHODS": ("GET", "POST"),
 326                              "ROLE_PERMISSION": "vnfds:",
 327                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 328                                       "ROLE_PERMISSION": "vnfds:id:",
 329                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 330                                                           "ROLE_PERMISSION": "vnfds:id:",
 331                                                           "upload_from_uri": {"METHODS": (),
 332                                                                               "TODO": ("POST", ),
 333                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 334                                                                               }
 335                                                           },
 336                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 337                                                "ROLE_PERMISSION": "vnfds:id:content:"
 338                                                },
 339                                       "artifacts": {"*": {"METHODS": ("GET", ),
 340                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 341                                                           }
 342                                                     },
 343                                       "action": {"METHODS": ("POST", ),
 344                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 345                                                  },
 346                                       }
 347                              },
 348             "subscriptions": {"TODO": ("GET", "POST"),
 349                               "<ID>": {"TODO": ("GET", "DELETE")}
 350                               },
 351             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 352                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 353                                "<ID>": {"METHODS": ("GET", ),
 354                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 355                                         }
 356                                },
 357         }
 358     },
 359     "nslcm": {
 360         "v1": {
 361             "ns_instances_content": {"METHODS": ("GET", "POST"),
 362                                      "ROLE_PERMISSION": "ns_instances:",
 363                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 364                                               "ROLE_PERMISSION": "ns_instances:id:"
 365                                               }
 366                                      },
 367             "ns_instances": {"METHODS": ("GET", "POST"),
 368                              "ROLE_PERMISSION": "ns_instances:",
 369                              "<ID>": {"METHODS": ("GET", "DELETE"),
 370                                       "ROLE_PERMISSION": "ns_instances:id:",
 371                                       "scale": {"METHODS": ("POST",),
 372                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 373                                                 },
 374                                       "terminate": {"METHODS": ("POST",),
 375                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 376                                                     },
 377                                       "instantiate": {"METHODS": ("POST",),
 378                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 379                                                       },
 380                                       "action": {"METHODS": ("POST",),
 381                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 382                                                  },
 383                                       }
 384                              },
 385             "ns_lcm_op_occs": {"METHODS": ("GET",),
 386                                "ROLE_PERMISSION": "ns_instances:opps:",
 387                                "<ID>": {"METHODS": ("GET",),
 388                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 389                                         },
 390                                },
 391             "vnfrs": {"METHODS": ("GET",),
 392                       "ROLE_PERMISSION": "vnf_instances:",
 393                       "<ID>": {"METHODS": ("GET",),
 394                                "ROLE_PERMISSION": "vnf_instances:id:"
 395                                }
 396                       },
 397             "vnf_instances": {"METHODS": ("GET",),
 398                               "ROLE_PERMISSION": "vnf_instances:",
 399                               "<ID>": {"METHODS": ("GET",),
 400                                        "ROLE_PERMISSION": "vnf_instances:id:"
 401                                        }
 402                               },
 403         }
 404     },
 405     "nst": {
 406         "v1": {
 407             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 408                                            "ROLE_PERMISSION": "slice_templates:",
 409                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 410                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 411                                            },
 412             "netslice_templates": {"METHODS": ("GET", "POST"),
 413                                    "ROLE_PERMISSION": "slice_templates:",
 414                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 415                                             "TODO": ("PATCH",),
 416                                             "ROLE_PERMISSION": "slice_templates:id:",
 417                                             "nst_content": {"METHODS": ("GET", "PUT"),
 418                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 419                                                             },
 420                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 421                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 422                                                     },
 423                                             "artifacts": {"*": {"METHODS": ("GET",),
 424                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 425                                                                 }
 426                                                           }
 427                                             }
 428                                    },
 429             "subscriptions": {"TODO": ("GET", "POST"),
 430                               "<ID>": {"TODO": ("GET", "DELETE")}
 431                               },
 432         }
 433     },
 434     "nsilcm": {
 435         "v1": {
 436             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 437                                            "ROLE_PERMISSION": "slice_instances:",
 438                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 439                                                     "ROLE_PERMISSION": "slice_instances:id:"
 440                                                     }
 441                                            },
 442             "netslice_instances": {"METHODS": ("GET", "POST"),
 443                                    "ROLE_PERMISSION": "slice_instances:",
 444                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 445                                             "ROLE_PERMISSION": "slice_instances:id:",
 446                                             "terminate": {"METHODS": ("POST",),
 447                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 448                                                           },
 449                                             "instantiate": {"METHODS": ("POST",),
 450                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 451                                                             },
 452                                             "action": {"METHODS": ("POST",),
 453                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 454                                                        },
 455                                             }
 456                                    },
 457             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 458                                 "ROLE_PERMISSION": "slice_instances:opps:",
 459                                 "<ID>": {"METHODS": ("GET",),
 460                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 461                                          },
 462                                 },
 463         }
 464     },
 465     "nspm": {
 466         "v1": {
 467             "pm_jobs": {
 468                 "<ID>": {
 469                     "reports": {
 470                         "<ID>": {"METHODS": ("GET",),
 471                                  "ROLE_PERMISSION": "reports:id:",
 472                                  }
 473                     }
 474                 },
 475             },
 476         },
 477     },
 478 }
 479
 480
 481 class NbiException(Exception):
 482
 483     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 484         Exception.__init__(self, message)
 485         self.http_code = http_code
 486
 487
 488 class Server(object):
 489     instance = 0
 490     # to decode bytes to str
 491     reader = getreader("utf-8")
 492
 493     def __init__(self):
 494         self.instance += 1
 495         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 496         self.engine = Engine(self.authenticator)
 497
 498     def _format_in(self, kwargs):
 499         try:
 500             indata = None
 501             if cherrypy.request.body.length:
 502                 error_text = "Invalid input format "
 503
 504                 if "Content-Type" in cherrypy.request.headers:
 505                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 506                         error_text = "Invalid json format "
 507                         indata = json.load(self.reader(cherrypy.request.body))
 508                         cherrypy.request.headers.pop("Content-File-MD5", None)
 509                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 510                         error_text = "Invalid yaml format "
 511                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 512                         cherrypy.request.headers.pop("Content-File-MD5", None)
 513                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 514                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 515                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 516                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 517                         indata = cherrypy.request.body  # .read()
 518                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 519                         if "descriptor_file" in kwargs:
 520                             filecontent = kwargs.pop("descriptor_file")
 521                             if not filecontent.file:
 522                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 523                             indata = filecontent.file  # .read()
 524                             if filecontent.content_type.value:
 525                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 526                     else:
 527                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 528                         #                          "Only 'Content-Type' of type 'application/json' or
 529                         # 'application/yaml' for input format are available")
 530                         error_text = "Invalid yaml format "
 531                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 532                         cherrypy.request.headers.pop("Content-File-MD5", None)
 533                 else:
 534                     error_text = "Invalid yaml format "
 535                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 536                     cherrypy.request.headers.pop("Content-File-MD5", None)
 537             if not indata:
 538                 indata = {}
 539
 540             format_yaml = False
 541             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 542                 format_yaml = True
 543
 544             for k, v in kwargs.items():
 545                 if isinstance(v, str):
 546                     if v == "":
 547                         kwargs[k] = None
 548                     elif format_yaml:
 549                         try:
 550                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 551                         except Exception:
 552                             pass
 553                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 554                         try:
 555                             kwargs[k] = int(v)
 556                         except Exception:
 557                             try:
 558                                 kwargs[k] = float(v)
 559                             except Exception:
 560                                 pass
 561                     elif v.find(",") > 0:
 562                         kwargs[k] = v.split(",")
 563                 elif isinstance(v, (list, tuple)):
 564                     for index in range(0, len(v)):
 565                         if v[index] == "":
 566                             v[index] = None
 567                         elif format_yaml:
 568                             try:
 569                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 570                             except Exception:
 571                                 pass
 572
 573             return indata
 574         except (ValueError, yaml.YAMLError) as exc:
 575             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 576         except KeyError as exc:
 577             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 578         except Exception as exc:
 579             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 580
 581     @staticmethod
 582     def _format_out(data, token_info=None, _format=None):
 583         """
 584         return string of dictionary data according to requested json, yaml, xml. By default json
 585         :param data: response to be sent. Can be a dict, text or file
 586         :param token_info: Contains among other username and project
 587         :param _format: The format to be set as Content-Type if data is a file
 588         :return: None
 589         """
 590         accept = cherrypy.request.headers.get("Accept")
 591         if data is None:
 592             if accept and "text/html" in accept:
 593                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 594             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 595             return
 596         elif hasattr(data, "read"):  # file object
 597             if _format:
 598                 cherrypy.response.headers["Content-Type"] = _format
 599             elif "b" in data.mode:  # binariy asssumig zip
 600                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 601             else:
 602                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 603             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 604             return data
 605         if accept:
 606             if "application/json" in accept:
 607                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 608                 a = json.dumps(data, indent=4) + "\n"
 609                 return a.encode("utf8")
 610             elif "text/html" in accept:
 611                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 612
 613             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 614                 pass
 615             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 616             elif cherrypy.response.status >= 400:
 617                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 618                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 619                                          "for output format are available")
 620         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 621         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 622                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 623
 624     @cherrypy.expose
 625     def index(self, *args, **kwargs):
 626         token_info = None
 627         try:
 628             if cherrypy.request.method == "GET":
 629                 token_info = self.authenticator.authorize()
 630                 outdata = token_info   # Home page
 631             else:
 632                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 633                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 634
 635             return self._format_out(outdata, token_info)
 636
 637         except (EngineException, AuthException) as e:
 638             # cherrypy.log("index Exception {}".format(e))
 639             cherrypy.response.status = e.http_code.value
 640             return self._format_out("Welcome to OSM!", token_info)
 641
 642     @cherrypy.expose
 643     def version(self, *args, **kwargs):
 644         # TODO consider to remove and provide version using the static version file
 645         try:
 646             if cherrypy.request.method != "GET":
 647                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 648             elif args or kwargs:
 649                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 650             # TODO include version of other modules, pick up from some kafka admin message
 651             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 652             return self._format_out(osm_nbi_version)
 653         except NbiException as e:
 654             cherrypy.response.status = e.http_code.value
 655             problem_details = {
 656                 "code": e.http_code.name,
 657                 "status": e.http_code.value,
 658                 "detail": str(e),
 659             }
 660             return self._format_out(problem_details, None)
 661
 662     def domain(self):
 663         try:
 664             domains = {
 665                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 666                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 667             return self._format_out(domains)
 668         except NbiException as e:
 669             cherrypy.response.status = e.http_code.value
 670             problem_details = {
 671                 "code": e.http_code.name,
 672                 "status": e.http_code.value,
 673                 "detail": str(e),
 674             }
 675             return self._format_out(problem_details, None)
 676
 677     @staticmethod
 678     def _format_login(token_info):
 679         """
 680         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 681         log this information
 682         :param token_info: Dictionary with token content
 683         :return: None
 684         """
 685         cherrypy.request.login = token_info.get("username", "-")
 686         if token_info.get("project_name"):
 687             cherrypy.request.login += "/" + token_info["project_name"]
 688         if token_info.get("id"):
 689             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 690
 691     @cherrypy.expose
 692     def token(self, method, token_id=None, kwargs=None):
 693         token_info = None
 694         # self.engine.load_dbase(cherrypy.request.app.config)
 695         indata = self._format_in(kwargs)
 696         if not isinstance(indata, dict):
 697             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 698
 699         if method == "GET":
 700             token_info = self.authenticator.authorize()
 701             # for logging
 702             self._format_login(token_info)
 703             if token_id:
 704                 outdata = self.authenticator.get_token(token_info, token_id)
 705             else:
 706                 outdata = self.authenticator.get_token_list(token_info)
 707         elif method == "POST":
 708             try:
 709                 token_info = self.authenticator.authorize()
 710             except Exception:
 711                 token_info = None
 712             if kwargs:
 713                 indata.update(kwargs)
 714             # This is needed to log the user when authentication fails
 715             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 716             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 717             cherrypy.session['Authorization'] = outdata["_id"]
 718             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 719             # for logging
 720             self._format_login(token_info)
 721
 722             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 723             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 724         elif method == "DELETE":
 725             if not token_id and "id" in kwargs:
 726                 token_id = kwargs["id"]
 727             elif not token_id:
 728                 token_info = self.authenticator.authorize()
 729                 # for logging
 730                 self._format_login(token_info)
 731                 token_id = token_info["_id"]
 732             outdata = self.authenticator.del_token(token_id)
 733             token_info = None
 734             cherrypy.session['Authorization'] = "logout"
 735             # cherrypy.response.cookie["Authorization"] = token_id
 736             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 737         else:
 738             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 739         return self._format_out(outdata, token_info)
 740
 741     @cherrypy.expose
 742     def test(self, *args, **kwargs):
 743         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 744                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 745             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 746             return "test URL is disabled"
 747         thread_info = None
 748         if args and args[0] == "help":
 749             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 750                    "sleep/<time>\nmessage/topic\n</pre></html>"
 751
 752         elif args and args[0] == "init":
 753             try:
 754                 # self.engine.load_dbase(cherrypy.request.app.config)
 755                 self.engine.create_admin()
 756                 return "Done. User 'admin', password 'admin' created"
 757             except Exception:
 758                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 759                 return self._format_out("Database already initialized")
 760         elif args and args[0] == "file":
 761             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 762                                                   "text/plain", "attachment")
 763         elif args and args[0] == "file2":
 764             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 765             f = open(f_path, "r")
 766             cherrypy.response.headers["Content-type"] = "text/plain"
 767             return f
 768
 769         elif len(args) == 2 and args[0] == "db-clear":
 770             deleted_info = self.engine.db.del_list(args[1], kwargs)
 771             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 772         elif len(args) and args[0] == "fs-clear":
 773             if len(args) >= 2:
 774                 folders = (args[1],)
 775             else:
 776                 folders = self.engine.fs.dir_ls(".")
 777             for folder in folders:
 778                 self.engine.fs.file_delete(folder)
 779             return ",".join(folders) + " folders deleted\n"
 780         elif args and args[0] == "login":
 781             if not cherrypy.request.headers.get("Authorization"):
 782                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 783                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 784         elif args and args[0] == "login2":
 785             if not cherrypy.request.headers.get("Authorization"):
 786                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 787                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 788         elif args and args[0] == "sleep":
 789             sleep_time = 5
 790             try:
 791                 sleep_time = int(args[1])
 792             except Exception:
 793                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 794                 return self._format_out("Database already initialized")
 795             thread_info = cherrypy.thread_data
 796             print(thread_info)
 797             time.sleep(sleep_time)
 798             # thread_info
 799         elif len(args) >= 2 and args[0] == "message":
 800             main_topic = args[1]
 801             return_text = "<html><pre>{} ->\n".format(main_topic)
 802             try:
 803                 if cherrypy.request.method == 'POST':
 804                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 805                     for k, v in to_send.items():
 806                         self.engine.msg.write(main_topic, k, v)
 807                         return_text += "  {}: {}\n".format(k, v)
 808                 elif cherrypy.request.method == 'GET':
 809                     for k, v in kwargs.items():
 810                         v_dict = yaml.load(v, Loader=yaml.SafeLoader)
 811                         self.engine.msg.write(main_topic, k, v_dict)
 812                         return_text += "  {}: {}\n".format(k, v_dict)
 813             except Exception as e:
 814                 return_text += "Error: " + str(e)
 815             return_text += "</pre></html>\n"
 816             return return_text
 817
 818         return_text = (
 819             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 820             "  kwargs: {}\n".format(kwargs) +
 821             "  headers: {}\n".format(cherrypy.request.headers) +
 822             "  path_info: {}\n".format(cherrypy.request.path_info) +
 823             "  query_string: {}\n".format(cherrypy.request.query_string) +
 824             "  session: {}\n".format(cherrypy.session) +
 825             "  cookie: {}\n".format(cherrypy.request.cookie) +
 826             "  method: {}\n".format(cherrypy.request.method) +
 827             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 828             "  body:\n")
 829         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 830         if cherrypy.request.body.length:
 831             return_text += "    content: {}\n".format(
 832                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 833         if thread_info:
 834             return_text += "thread: {}\n".format(thread_info)
 835         return_text += "</pre></html>"
 836         return return_text
 837
 838     @staticmethod
 839     def _check_valid_url_method(method, *args):
 840         if len(args) < 3:
 841             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 842
 843         reference = valid_url_methods
 844         for arg in args:
 845             if arg is None:
 846                 break
 847             if not isinstance(reference, dict):
 848                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 849                                    HTTPStatus.METHOD_NOT_ALLOWED)
 850
 851             if arg in reference:
 852                 reference = reference[arg]
 853             elif "<ID>" in reference:
 854                 reference = reference["<ID>"]
 855             elif "*" in reference:
 856                 reference = reference["*"]
 857                 break
 858             else:
 859                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 860         if "TODO" in reference and method in reference["TODO"]:
 861             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 862         elif "METHODS" in reference and method not in reference["METHODS"]:
 863             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 864         return reference["ROLE_PERMISSION"] + method.lower()
 865
 866     @staticmethod
 867     def _set_location_header(main_topic, version, topic, id):
 868         """
 869         Insert response header Location with the URL of created item base on URL params
 870         :param main_topic:
 871         :param version:
 872         :param topic:
 873         :param id:
 874         :return: None
 875         """
 876         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 877         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 878         return
 879
 880     @staticmethod
 881     def _extract_query_string_operations(kwargs, method):
 882         """
 883
 884         :param kwargs:
 885         :return:
 886         """
 887         query_string_operations = []
 888         if kwargs:
 889             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 890                 if qs in kwargs and kwargs[qs].lower() != "false":
 891                     query_string_operations.append(qs.lower() + ":" + method.lower())
 892         return query_string_operations
 893
 894     @staticmethod
 895     def _manage_admin_query(token_info, kwargs, method, _id):
 896         """
 897         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 898         Check that users has rights to use them and returs the admin_query
 899         :param token_info: token_info rights obtained by token
 900         :param kwargs: query string input.
 901         :param method: http method: GET, POSST, PUT, ...
 902         :param _id:
 903         :return: admin_query dictionary with keys:
 904             public: True, False or None
 905             force: True or False
 906             project_id: tuple with projects used for accessing an element
 907             set_project: tuple with projects that a created element will belong to
 908             method: show, list, delete, write
 909         """
 910         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 911                        "admin": token_info["admin"], "public": None,
 912                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 913         if kwargs:
 914             # FORCE
 915             if "FORCE" in kwargs:
 916                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 917                     admin_query["force"] = True
 918                 del kwargs["FORCE"]
 919             # PUBLIC
 920             if "PUBLIC" in kwargs:
 921                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 922                     admin_query["public"] = True
 923                 else:
 924                     admin_query["public"] = False
 925                 del kwargs["PUBLIC"]
 926             # ADMIN
 927             if "ADMIN" in kwargs:
 928                 behave_as = kwargs.pop("ADMIN")
 929                 if behave_as.lower() != "false":
 930                     if not token_info["admin"]:
 931                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 932                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 933                         admin_query["project_id"] = ()
 934                     elif isinstance(behave_as, (list, tuple)):
 935                         admin_query["project_id"] = behave_as
 936                     else:   # isinstance(behave_as, str)
 937                         admin_query["project_id"] = (behave_as, )
 938             if "SET_PROJECT" in kwargs:
 939                 set_project = kwargs.pop("SET_PROJECT")
 940                 if not set_project:
 941                     admin_query["set_project"] = list(admin_query["project_id"])
 942                 else:
 943                     if isinstance(set_project, str):
 944                         set_project = (set_project, )
 945                     if admin_query["project_id"]:
 946                         for p in set_project:
 947                             if p not in admin_query["project_id"]:
 948                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 949                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 950                     admin_query["set_project"] = set_project
 951
 952             # PROJECT_READ
 953             # if "PROJECT_READ" in kwargs:
 954             #     admin_query["project"] = kwargs.pop("project")
 955             #     if admin_query["project"] == token_info["project_id"]:
 956         if method == "GET":
 957             if _id:
 958                 admin_query["method"] = "show"
 959             else:
 960                 admin_query["method"] = "list"
 961         elif method == "DELETE":
 962             admin_query["method"] = "delete"
 963         else:
 964             admin_query["method"] = "write"
 965         return admin_query
 966
 967     @cherrypy.expose
 968     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 969         token_info = None
 970         outdata = None
 971         _format = None
 972         method = "DONE"
 973         engine_topic = None
 974         rollback = []
 975         engine_session = None
 976         try:
 977             if not main_topic or not version or not topic:
 978                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 979                                    HTTPStatus.METHOD_NOT_ALLOWED)
 980             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 981                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 982                                    HTTPStatus.METHOD_NOT_ALLOWED)
 983             if version != 'v1':
 984                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 985
 986             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 987                 method = kwargs.pop("METHOD")
 988             else:
 989                 method = cherrypy.request.method
 990
 991             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 992             query_string_operations = self._extract_query_string_operations(kwargs, method)
 993             if main_topic == "admin" and topic == "tokens":
 994                 return self.token(method, _id, kwargs)
 995             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
 996             if main_topic == "admin" and topic == "domains":
 997                 return self.domain()
 998             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 999             indata = self._format_in(kwargs)
1000             engine_topic = topic
1001             if topic == "subscriptions":
1002                 engine_topic = main_topic + "_" + topic
1003             if item and topic != "pm_jobs":
1004                 engine_topic = item
1005
1006             if main_topic == "nsd":
1007                 engine_topic = "nsds"
1008             elif main_topic == "vnfpkgm":
1009                 engine_topic = "vnfds"
1010                 if topic == "vnfpkg_op_occs":
1011                     engine_topic = "vnfpkgops"
1012                 if topic == "vnf_packages" and item == "action":
1013                     engine_topic = "vnfpkgops"
1014             elif main_topic == "nslcm":
1015                 engine_topic = "nsrs"
1016                 if topic == "ns_lcm_op_occs":
1017                     engine_topic = "nslcmops"
1018                 if topic == "vnfrs" or topic == "vnf_instances":
1019                     engine_topic = "vnfrs"
1020             elif main_topic == "nst":
1021                 engine_topic = "nsts"
1022             elif main_topic == "nsilcm":
1023                 engine_topic = "nsis"
1024                 if topic == "nsi_lcm_op_occs":
1025                     engine_topic = "nsilcmops"
1026             elif main_topic == "pdu":
1027                 engine_topic = "pdus"
1028             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1029                 engine_topic = "vim_accounts"
1030
1031             if method == "GET":
1032                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1033                     if item in ("vnfd", "nsd", "nst"):
1034                         path = "$DESCRIPTOR"
1035                     elif args:
1036                         path = args
1037                     elif item == "artifacts":
1038                         path = ()
1039                     else:
1040                         path = None
1041                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1042                                                          cherrypy.request.headers.get("Accept"))
1043                     outdata = file
1044                 elif not _id:
1045                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1046                 else:
1047                     if item == "reports":
1048                         # TODO check that project_id (_id in this context) has permissions
1049                         _id = args[0]
1050                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1051
1052             elif method == "POST":
1053                 cherrypy.response.status = HTTPStatus.CREATED.value
1054                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1055                     _id = cherrypy.request.headers.get("Transaction-Id")
1056                     if not _id:
1057                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1058                                                       cherrypy.request.headers)
1059                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1060                                                            cherrypy.request.headers)
1061                     if completed:
1062                         self._set_location_header(main_topic, version, topic, _id)
1063                     else:
1064                         cherrypy.response.headers["Transaction-Id"] = _id
1065                     outdata = {"id": _id}
1066                 elif topic == "ns_instances_content":
1067                     # creates NSR
1068                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1069                     # creates nslcmop
1070                     indata["lcmOperationType"] = "instantiate"
1071                     indata["nsInstanceId"] = _id
1072                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1073                     self._set_location_header(main_topic, version, topic, _id)
1074                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1075                 elif topic == "ns_instances" and item:
1076                     indata["lcmOperationType"] = item
1077                     indata["nsInstanceId"] = _id
1078                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1079                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1080                     outdata = {"id": _id}
1081                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1082                 elif topic == "netslice_instances_content":
1083                     # creates NetSlice_Instance_record (NSIR)
1084                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1085                     self._set_location_header(main_topic, version, topic, _id)
1086                     indata["lcmOperationType"] = "instantiate"
1087                     indata["netsliceInstanceId"] = _id
1088                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1089                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1090                 elif topic == "netslice_instances" and item:
1091                     indata["lcmOperationType"] = item
1092                     indata["netsliceInstanceId"] = _id
1093                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1094                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1095                     outdata = {"id": _id}
1096                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1097                 elif topic == "vnf_packages" and item == "action":
1098                     indata["lcmOperationType"] = item
1099                     indata["vnfPkgId"] = _id
1100                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1101                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1102                     outdata = {"id": _id}
1103                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1104                 else:
1105                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1106                                                       cherrypy.request.headers)
1107                     self._set_location_header(main_topic, version, topic, _id)
1108                     outdata = {"id": _id}
1109                     if op_id:
1110                         outdata["op_id"] = op_id
1111                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1112                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1113
1114             elif method == "DELETE":
1115                 if not _id:
1116                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1117                     cherrypy.response.status = HTTPStatus.OK.value
1118                 else:  # len(args) > 1
1119                     # for NS NSI generate an operation
1120                     op_id = None
1121                     if topic == "ns_instances_content" and not engine_session["force"]:
1122                         nslcmop_desc = {
1123                             "lcmOperationType": "terminate",
1124                             "nsInstanceId": _id,
1125                             "autoremove": True
1126                         }
1127                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1128                         if op_id:
1129                             outdata = {"_id": op_id}
1130                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1131                         nsilcmop_desc = {
1132                             "lcmOperationType": "terminate",
1133                             "netsliceInstanceId": _id,
1134                             "autoremove": True
1135                         }
1136                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1137                         if op_id:
1138                             outdata = {"_id": op_id}
1139                     # if there is not any deletion in process, delete
1140                     if not op_id:
1141                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1142                         if op_id:
1143                             outdata = {"op_id": op_id}
1144                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1145
1146             elif method in ("PUT", "PATCH"):
1147                 op_id = None
1148                 if not indata and not kwargs and not engine_session.get("set_project"):
1149                     raise NbiException("Nothing to update. Provide payload and/or query string",
1150                                        HTTPStatus.BAD_REQUEST)
1151                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1152                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1153                                                            cherrypy.request.headers)
1154                     if not completed:
1155                         cherrypy.response.headers["Transaction-Id"] = id
1156                 else:
1157                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1158
1159                 if op_id:
1160                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1161                     outdata = {"op_id": op_id}
1162                 else:
1163                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1164                     outdata = None
1165             else:
1166                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1167
1168             # if Role information changes, it is needed to reload the information of roles
1169             if topic == "roles" and method != "GET":
1170                 self.authenticator.load_operation_to_allowed_roles()
1171
1172             if topic == "projects" and method == "DELETE" \
1173                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1174                 self.authenticator.remove_token_from_cache()
1175
1176             return self._format_out(outdata, token_info, _format)
1177         except Exception as e:
1178             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1179                               ValidationError, AuthconnException)):
1180                 http_code_value = cherrypy.response.status = e.http_code.value
1181                 http_code_name = e.http_code.name
1182                 cherrypy.log("Exception {}".format(e))
1183             else:
1184                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1185                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1186                 http_code_name = HTTPStatus.BAD_REQUEST.name
1187             if hasattr(outdata, "close"):  # is an open file
1188                 outdata.close()
1189             error_text = str(e)
1190             rollback.reverse()
1191             for rollback_item in rollback:
1192                 try:
1193                     if rollback_item.get("operation") == "set":
1194                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1195                                                rollback_item["content"], fail_on_empty=False)
1196                     else:
1197                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1198                                                fail_on_empty=False)
1199                 except Exception as e2:
1200                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1201                     cherrypy.log(rollback_error_text)
1202                     error_text += ". " + rollback_error_text
1203             # if isinstance(e, MsgException):
1204             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1205             #         engine_topic[:-1], method, error_text)
1206             problem_details = {
1207                 "code": http_code_name,
1208                 "status": http_code_value,
1209                 "detail": error_text,
1210             }
1211             return self._format_out(problem_details, token_info)
1212             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1213         finally:
1214             if token_info:
1215                 self._format_login(token_info)
1216                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1217                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1218                         if outdata.get(logging_id):
1219                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1220
1221
1222 def _start_service():
1223     """
1224     Callback function called when cherrypy.engine starts
1225     Override configuration with env variables
1226     Set database, storage, message configuration
1227     Init database with admin/admin user password
1228     """
1229     global nbi_server
1230     global subscription_thread
1231     cherrypy.log.error("Starting osm_nbi")
1232     # update general cherrypy configuration
1233     update_dict = {}
1234
1235     engine_config = cherrypy.tree.apps['/osm'].config
1236     for k, v in environ.items():
1237         if not k.startswith("OSMNBI_"):
1238             continue
1239         k1, _, k2 = k[7:].lower().partition("_")
1240         if not k2:
1241             continue
1242         try:
1243             # update static configuration
1244             if k == 'OSMNBI_STATIC_DIR':
1245                 engine_config["/static"]['tools.staticdir.dir'] = v
1246                 engine_config["/static"]['tools.staticdir.on'] = True
1247             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1248                 update_dict['server.socket_port'] = int(v)
1249             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1250                 update_dict['server.socket_host'] = v
1251             elif k1 in ("server", "test", "auth", "log"):
1252                 update_dict[k1 + '.' + k2] = v
1253             elif k1 in ("message", "database", "storage", "authentication"):
1254                 # k2 = k2.replace('_', '.')
1255                 if k2 in ("port", "db_port"):
1256                     engine_config[k1][k2] = int(v)
1257                 else:
1258                     engine_config[k1][k2] = v
1259
1260         except ValueError as e:
1261             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1262         except Exception as e:
1263             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1264
1265     if update_dict:
1266         cherrypy.config.update(update_dict)
1267         engine_config["global"].update(update_dict)
1268
1269     # logging cherrypy
1270     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1271     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1272     logger_server = logging.getLogger("cherrypy.error")
1273     logger_access = logging.getLogger("cherrypy.access")
1274     logger_cherry = logging.getLogger("cherrypy")
1275     logger_nbi = logging.getLogger("nbi")
1276
1277     if "log.file" in engine_config["global"]:
1278         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1279                                                             maxBytes=100e6, backupCount=9, delay=0)
1280         file_handler.setFormatter(log_formatter_simple)
1281         logger_cherry.addHandler(file_handler)
1282         logger_nbi.addHandler(file_handler)
1283     # log always to standard output
1284     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1285                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1286                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1287                             }.items():
1288         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1289         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1290         str_handler = logging.StreamHandler()
1291         str_handler.setFormatter(log_formatter_cherry)
1292         logger.addHandler(str_handler)
1293
1294     if engine_config["global"].get("log.level"):
1295         logger_cherry.setLevel(engine_config["global"]["log.level"])
1296         logger_nbi.setLevel(engine_config["global"]["log.level"])
1297
1298     # logging other modules
1299     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1300         engine_config[k1]["logger_name"] = logname
1301         logger_module = logging.getLogger(logname)
1302         if "logfile" in engine_config[k1]:
1303             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1304                                                                 maxBytes=100e6, backupCount=9, delay=0)
1305             file_handler.setFormatter(log_formatter_simple)
1306             logger_module.addHandler(file_handler)
1307         if "loglevel" in engine_config[k1]:
1308             logger_module.setLevel(engine_config[k1]["loglevel"])
1309     # TODO add more entries, e.g.: storage
1310     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1311     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1312     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1313     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1314
1315     # start subscriptions thread:
1316     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1317     subscription_thread.start()
1318     # Do not capture except SubscriptionException
1319
1320     backend = engine_config["authentication"]["backend"]
1321     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1322                        .format(nbi_version, nbi_version_date, backend))
1323
1324
1325 def _stop_service():
1326     """
1327     Callback function called when cherrypy.engine stops
1328     TODO: Ending database connections.
1329     """
1330     global subscription_thread
1331     if subscription_thread:
1332         subscription_thread.terminate()
1333     subscription_thread = None
1334     cherrypy.tree.apps['/osm'].root.engine.stop()
1335     cherrypy.log.error("Stopping osm_nbi")
1336
1337
1338 def nbi(config_file):
1339     global nbi_server
1340     # conf = {
1341     #     '/': {
1342     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1343     #         'tools.sessions.on': True,
1344     #         'tools.response_headers.on': True,
1345     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1346     #     }
1347     # }
1348     # cherrypy.Server.ssl_module = 'builtin'
1349     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1350     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1351     # cherrypy.Server.thread_pool = 10
1352     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1353
1354     # cherrypy.config.update({'tools.auth_basic.on': True,
1355     #    'tools.auth_basic.realm': 'localhost',
1356     #    'tools.auth_basic.checkpassword': validate_password})
1357     nbi_server = Server()
1358     cherrypy.engine.subscribe('start', _start_service)
1359     cherrypy.engine.subscribe('stop', _stop_service)
1360     cherrypy.quickstart(nbi_server, '/osm', config_file)
1361
1362
1363 def usage():
1364     print("""Usage: {} [options]
1365         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1366         -h|--help: shows this help
1367         """.format(sys.argv[0]))
1368     # --log-socket-host HOST: send logs to this host")
1369     # --log-socket-port PORT: send logs using this port (default: 9022)")
1370
1371
1372 if __name__ == '__main__':
1373     try:
1374         # load parameters and configuration
1375         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1376         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1377         config_file = None
1378         for o, a in opts:
1379             if o in ("-h", "--help"):
1380                 usage()
1381                 sys.exit()
1382             elif o in ("-c", "--config"):
1383                 config_file = a
1384             # elif o == "--log-socket-port":
1385             #     log_socket_port = a
1386             # elif o == "--log-socket-host":
1387             #     log_socket_host = a
1388             # elif o == "--log-file":
1389             #     log_file = a
1390             else:
1391                 assert False, "Unhandled option"
1392         if config_file:
1393             if not path.isfile(config_file):
1394                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1395                 exit(1)
1396         else:
1397             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1398                 if path.isfile(config_file):
1399                     break
1400             else:
1401                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1402                 exit(1)
1403         nbi(config_file)
1404     except getopt.GetoptError as e:
1405         print(str(e), file=sys.stderr)
1406         # usage()
1407         exit(1)