osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118
 119         /nst/v1                                                 O       O
 120             /netslice_templates_content                         O       O
 121                 /<nstInfoId>                                    O       O       O       O
 122             /netslice_templates                                 O       O
 123                 /<nstInfoId>                                    O                       O       O
 124                     /nst_content                                O               O
 125                     /nst                                        O
 126                     /artifacts[/<artifactPath>]                 O
 127             /subscriptions                                      X       X
 128                 /<subscriptionId>                               X                       X
 129
 130         /nsilcm/v1
 131             /netslice_instances_content                         O       O
 132                 /<SliceInstanceId>                              O                       O
 133             /netslice_instances                                 O       O
 134                 /<SliceInstanceId>                              O                       O
 135                     instantiate                                         O
 136                     terminate                                           O
 137                     action                                              O
 138             /nsi_lcm_op_occs                                    O       O
 139                 /<nsiLcmOpOccId>                                O                       O       O
 140             /subscriptions                                      X       X
 141                 /<subscriptionId>                               X                       X
 142
 143 query string:
 144     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 145         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 146         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 147         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 148         attrName := string
 149     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 150     item of the array, that is, pass if any item of the array pass the filter.
 151     It allows both ne and neq for not equal
 152     TODO: 4.3.3 Attribute selectors
 153         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 154         (none)  … same as “exclude_default”
 155         all_fields      … all attributes.
 156         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 157         conditionally mandatory, and that are not provided in <list>.
 158         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 159         are not conditionally mandatory, and that are provided in <list>.
 160         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 161         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 162         the particular resource
 163         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 164         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 165         present specification for the particular resource, but that are not part of <list>
 166     Additionally it admits some administrator values:
 167         FORCE: To force operations skipping dependency checkings
 168         ADMIN: To act as an administrator or a different project
 169         PUBLIC: To get public descriptors or set a descriptor as public
 170         SET_PROJECT: To make a descriptor available for other project
 171
 172 Header field name       Reference       Example Descriptions
 173     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 174     This header field shall be present if the response is expected to have a non-empty message body.
 175     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 176     This header field shall be present if the request has a non-empty message body.
 177     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 178     Details are specified in clause 4.5.3.
 179     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 180 Header field name       Reference       Example Descriptions
 181     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 182     This header field shall be present if the response has a non-empty message body.
 183     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 184     new resource has been created.
 185     This header field shall be present if the response status code is 201 or 3xx.
 186     In the present document this header field is also used if the response status code is 202 and a new resource was
 187     created.
 188     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 189     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 190     token.
 191     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 192     certain resources.
 193     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 194     response, and the total length of the file.
 195     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 196 """
 197
 198 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 199 # ^ Contains possible administrative query string words:
 200 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 201 #           (not owned by my session project).
 202 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 203 #     FORCE=True(by default)|False: Force edition/deletion operations
 204 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 205
 206 valid_url_methods = {
 207     # contains allowed URL and methods, and the role_permission name
 208     "admin": {
 209         "v1": {
 210             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 211                        "ROLE_PERMISSION": "tokens:",
 212                        "<ID>": {"METHODS": ("GET", "DELETE"),
 213                                 "ROLE_PERMISSION": "tokens:id:"
 214                                 }
 215                        },
 216             "users": {"METHODS": ("GET", "POST"),
 217                       "ROLE_PERMISSION": "users:",
 218                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 219                                "ROLE_PERMISSION": "users:id:"
 220                                }
 221                       },
 222             "projects": {"METHODS": ("GET", "POST"),
 223                          "ROLE_PERMISSION": "projects:",
 224                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 225                                   "ROLE_PERMISSION": "projects:id:"}
 226                          },
 227             "roles": {"METHODS": ("GET", "POST"),
 228                       "ROLE_PERMISSION": "roles:",
 229                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 230                                "ROLE_PERMISSION": "roles:id:"
 231                                }
 232                       },
 233             "vims": {"METHODS": ("GET", "POST"),
 234                      "ROLE_PERMISSION": "vims:",
 235                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 236                               "ROLE_PERMISSION": "vims:id:"
 237                               }
 238                      },
 239             "vim_accounts": {"METHODS": ("GET", "POST"),
 240                              "ROLE_PERMISSION": "vim_accounts:",
 241                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 242                                       "ROLE_PERMISSION": "vim_accounts:id:"
 243                                       }
 244                              },
 245             "wim_accounts": {"METHODS": ("GET", "POST"),
 246                              "ROLE_PERMISSION": "wim_accounts:",
 247                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 248                                       "ROLE_PERMISSION": "wim_accounts:id:"
 249                                       }
 250                              },
 251             "sdns": {"METHODS": ("GET", "POST"),
 252                      "ROLE_PERMISSION": "sdn_controllers:",
 253                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 254                               "ROLE_PERMISSION": "sdn_controllers:id:"
 255                               }
 256                      },
 257             "k8sclusters": {"METHODS": ("GET", "POST"),
 258                             "ROLE_PERMISSION": "k8sclusters:",
 259                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 260                                      "ROLE_PERMISSION": "k8sclusters:id:"
 261                                      }
 262                             },
 263             "k8srepos": {"METHODS": ("GET", "POST"),
 264                          "ROLE_PERMISSION": "k8srepos:",
 265                          "<ID>": {"METHODS": ("GET", "DELETE"),
 266                                   "ROLE_PERMISSION": "k8srepos:id:"
 267                                   }
 268                          },
 269             "domains": {"METHODS": ("GET", ),
 270                         "ROLE_PERMISSION": "domains:",
 271                         },
 272         }
 273     },
 274     "pdu": {
 275         "v1": {
 276             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 277                                 "ROLE_PERMISSION": "pduds:",
 278                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 279                                          "ROLE_PERMISSION": "pduds:id:"
 280                                          }
 281                                 },
 282         }
 283     },
 284     "nsd": {
 285         "v1": {
 286             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 287                                        "ROLE_PERMISSION": "nsds:",
 288                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 289                                                 "ROLE_PERMISSION": "nsds:id:"
 290                                                 }
 291                                        },
 292             "ns_descriptors": {"METHODS": ("GET", "POST"),
 293                                "ROLE_PERMISSION": "nsds:",
 294                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 295                                         "ROLE_PERMISSION": "nsds:id:",
 296                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 297                                                         "ROLE_PERMISSION": "nsds:id:content:",
 298                                                         },
 299                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 300                                                 "ROLE_PERMISSION": "nsds:id:content:"
 301                                                 },
 302                                         "artifacts": {"*": {"METHODS": ("GET",),
 303                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 304                                                             }
 305                                                       }
 306                                         }
 307                                },
 308             "pnf_descriptors": {"TODO": ("GET", "POST"),
 309                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 310                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 311                                          }
 312                                 },
 313             "subscriptions": {"TODO": ("GET", "POST"),
 314                               "<ID>": {"TODO": ("GET", "DELETE")}
 315                               },
 316         }
 317     },
 318     "vnfpkgm": {
 319         "v1": {
 320             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 321                                      "ROLE_PERMISSION": "vnfds:",
 322                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 323                                               "ROLE_PERMISSION": "vnfds:id:"}
 324                                      },
 325             "vnf_packages": {"METHODS": ("GET", "POST"),
 326                              "ROLE_PERMISSION": "vnfds:",
 327                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 328                                       "ROLE_PERMISSION": "vnfds:id:",
 329                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 330                                                           "ROLE_PERMISSION": "vnfds:id:",
 331                                                           "upload_from_uri": {"METHODS": (),
 332                                                                               "TODO": ("POST", ),
 333                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 334                                                                               }
 335                                                           },
 336                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 337                                                "ROLE_PERMISSION": "vnfds:id:content:"
 338                                                },
 339                                       "artifacts": {"*": {"METHODS": ("GET", ),
 340                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 341                                                           }
 342                                                     },
 343                                       "action": {"METHODS": ("POST", ),
 344                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 345                                                  },
 346                                       }
 347                              },
 348             "subscriptions": {"TODO": ("GET", "POST"),
 349                               "<ID>": {"TODO": ("GET", "DELETE")}
 350                               },
 351             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 352                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 353                                "<ID>": {"METHODS": ("GET", ),
 354                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 355                                         }
 356                                },
 357         }
 358     },
 359     "nslcm": {
 360         "v1": {
 361             "ns_instances_content": {"METHODS": ("GET", "POST"),
 362                                      "ROLE_PERMISSION": "ns_instances:",
 363                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 364                                               "ROLE_PERMISSION": "ns_instances:id:"
 365                                               }
 366                                      },
 367             "ns_instances": {"METHODS": ("GET", "POST"),
 368                              "ROLE_PERMISSION": "ns_instances:",
 369                              "<ID>": {"METHODS": ("GET", "DELETE"),
 370                                       "ROLE_PERMISSION": "ns_instances:id:",
 371                                       "scale": {"METHODS": ("POST",),
 372                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 373                                                 },
 374                                       "terminate": {"METHODS": ("POST",),
 375                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 376                                                     },
 377                                       "instantiate": {"METHODS": ("POST",),
 378                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 379                                                       },
 380                                       "action": {"METHODS": ("POST",),
 381                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 382                                                  },
 383                                       }
 384                              },
 385             "ns_lcm_op_occs": {"METHODS": ("GET",),
 386                                "ROLE_PERMISSION": "ns_instances:opps:",
 387                                "<ID>": {"METHODS": ("GET",),
 388                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 389                                         },
 390                                },
 391             "vnfrs": {"METHODS": ("GET",),
 392                       "ROLE_PERMISSION": "vnf_instances:",
 393                       "<ID>": {"METHODS": ("GET",),
 394                                "ROLE_PERMISSION": "vnf_instances:id:"
 395                                }
 396                       },
 397             "vnf_instances": {"METHODS": ("GET",),
 398                               "ROLE_PERMISSION": "vnf_instances:",
 399                               "<ID>": {"METHODS": ("GET",),
 400                                        "ROLE_PERMISSION": "vnf_instances:id:"
 401                                        }
 402                               },
 403         }
 404     },
 405     "nst": {
 406         "v1": {
 407             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 408                                            "ROLE_PERMISSION": "slice_templates:",
 409                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 410                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 411                                            },
 412             "netslice_templates": {"METHODS": ("GET", "POST"),
 413                                    "ROLE_PERMISSION": "slice_templates:",
 414                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 415                                             "TODO": ("PATCH",),
 416                                             "ROLE_PERMISSION": "slice_templates:id:",
 417                                             "nst_content": {"METHODS": ("GET", "PUT"),
 418                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 419                                                             },
 420                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 421                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 422                                                     },
 423                                             "artifacts": {"*": {"METHODS": ("GET",),
 424                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 425                                                                 }
 426                                                           }
 427                                             }
 428                                    },
 429             "subscriptions": {"TODO": ("GET", "POST"),
 430                               "<ID>": {"TODO": ("GET", "DELETE")}
 431                               },
 432         }
 433     },
 434     "nsilcm": {
 435         "v1": {
 436             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 437                                            "ROLE_PERMISSION": "slice_instances:",
 438                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 439                                                     "ROLE_PERMISSION": "slice_instances:id:"
 440                                                     }
 441                                            },
 442             "netslice_instances": {"METHODS": ("GET", "POST"),
 443                                    "ROLE_PERMISSION": "slice_instances:",
 444                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 445                                             "ROLE_PERMISSION": "slice_instances:id:",
 446                                             "terminate": {"METHODS": ("POST",),
 447                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 448                                                           },
 449                                             "instantiate": {"METHODS": ("POST",),
 450                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 451                                                             },
 452                                             "action": {"METHODS": ("POST",),
 453                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 454                                                        },
 455                                             }
 456                                    },
 457             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 458                                 "ROLE_PERMISSION": "slice_instances:opps:",
 459                                 "<ID>": {"METHODS": ("GET",),
 460                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 461                                          },
 462                                 },
 463         }
 464     },
 465     "nspm": {
 466         "v1": {
 467             "pm_jobs": {
 468                 "<ID>": {
 469                     "reports": {
 470                         "<ID>": {"METHODS": ("GET",),
 471                                  "ROLE_PERMISSION": "reports:id:",
 472                                  }
 473                     }
 474                 },
 475             },
 476         },
 477     },
 478 }
 479
 480
 481 class NbiException(Exception):
 482
 483     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 484         Exception.__init__(self, message)
 485         self.http_code = http_code
 486
 487
 488 class Server(object):
 489     instance = 0
 490     # to decode bytes to str
 491     reader = getreader("utf-8")
 492
 493     def __init__(self):
 494         self.instance += 1
 495         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 496         self.engine = Engine(self.authenticator)
 497
 498     def _format_in(self, kwargs):
 499         try:
 500             indata = None
 501             if cherrypy.request.body.length:
 502                 error_text = "Invalid input format "
 503
 504                 if "Content-Type" in cherrypy.request.headers:
 505                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 506                         error_text = "Invalid json format "
 507                         indata = json.load(self.reader(cherrypy.request.body))
 508                         cherrypy.request.headers.pop("Content-File-MD5", None)
 509                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 510                         error_text = "Invalid yaml format "
 511                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 512                         cherrypy.request.headers.pop("Content-File-MD5", None)
 513                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 514                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 515                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 516                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 517                         indata = cherrypy.request.body  # .read()
 518                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 519                         if "descriptor_file" in kwargs:
 520                             filecontent = kwargs.pop("descriptor_file")
 521                             if not filecontent.file:
 522                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 523                             indata = filecontent.file  # .read()
 524                             if filecontent.content_type.value:
 525                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 526                     else:
 527                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 528                         #                          "Only 'Content-Type' of type 'application/json' or
 529                         # 'application/yaml' for input format are available")
 530                         error_text = "Invalid yaml format "
 531                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 532                         cherrypy.request.headers.pop("Content-File-MD5", None)
 533                 else:
 534                     error_text = "Invalid yaml format "
 535                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 536                     cherrypy.request.headers.pop("Content-File-MD5", None)
 537             if not indata:
 538                 indata = {}
 539
 540             format_yaml = False
 541             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 542                 format_yaml = True
 543
 544             for k, v in kwargs.items():
 545                 if isinstance(v, str):
 546                     if v == "":
 547                         kwargs[k] = None
 548                     elif format_yaml:
 549                         try:
 550                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 551                         except Exception:
 552                             pass
 553                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 554                         try:
 555                             kwargs[k] = int(v)
 556                         except Exception:
 557                             try:
 558                                 kwargs[k] = float(v)
 559                             except Exception:
 560                                 pass
 561                     elif v.find(",") > 0:
 562                         kwargs[k] = v.split(",")
 563                 elif isinstance(v, (list, tuple)):
 564                     for index in range(0, len(v)):
 565                         if v[index] == "":
 566                             v[index] = None
 567                         elif format_yaml:
 568                             try:
 569                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 570                             except Exception:
 571                                 pass
 572
 573             return indata
 574         except (ValueError, yaml.YAMLError) as exc:
 575             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 576         except KeyError as exc:
 577             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 578         except Exception as exc:
 579             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 580
 581     @staticmethod
 582     def _format_out(data, token_info=None, _format=None):
 583         """
 584         return string of dictionary data according to requested json, yaml, xml. By default json
 585         :param data: response to be sent. Can be a dict, text or file
 586         :param token_info: Contains among other username and project
 587         :param _format: The format to be set as Content-Type if data is a file
 588         :return: None
 589         """
 590         accept = cherrypy.request.headers.get("Accept")
 591         if data is None:
 592             if accept and "text/html" in accept:
 593                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 594             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 595             return
 596         elif hasattr(data, "read"):  # file object
 597             if _format:
 598                 cherrypy.response.headers["Content-Type"] = _format
 599             elif "b" in data.mode:  # binariy asssumig zip
 600                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 601             else:
 602                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 603             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 604             return data
 605         if accept:
 606             if "application/json" in accept:
 607                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 608                 a = json.dumps(data, indent=4) + "\n"
 609                 return a.encode("utf8")
 610             elif "text/html" in accept:
 611                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 612
 613             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 614                 pass
 615             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 616             elif cherrypy.response.status >= 400:
 617                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 618                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 619                                          "for output format are available")
 620         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 621         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 622                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 623
 624     @cherrypy.expose
 625     def index(self, *args, **kwargs):
 626         token_info = None
 627         try:
 628             if cherrypy.request.method == "GET":
 629                 token_info = self.authenticator.authorize()
 630                 outdata = token_info   # Home page
 631             else:
 632                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 633                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 634
 635             return self._format_out(outdata, token_info)
 636
 637         except (EngineException, AuthException) as e:
 638             # cherrypy.log("index Exception {}".format(e))
 639             cherrypy.response.status = e.http_code.value
 640             return self._format_out("Welcome to OSM!", token_info)
 641
 642     @cherrypy.expose
 643     def version(self, *args, **kwargs):
 644         # TODO consider to remove and provide version using the static version file
 645         try:
 646             if cherrypy.request.method != "GET":
 647                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 648             elif args or kwargs:
 649                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 650             # TODO include version of other modules, pick up from some kafka admin message
 651             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 652             return self._format_out(osm_nbi_version)
 653         except NbiException as e:
 654             cherrypy.response.status = e.http_code.value
 655             problem_details = {
 656                 "code": e.http_code.name,
 657                 "status": e.http_code.value,
 658                 "detail": str(e),
 659             }
 660             return self._format_out(problem_details, None)
 661
 662     def domain(self):
 663         try:
 664             domains = {
 665                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 666                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 667             return self._format_out(domains)
 668         except NbiException as e:
 669             cherrypy.response.status = e.http_code.value
 670             problem_details = {
 671                 "code": e.http_code.name,
 672                 "status": e.http_code.value,
 673                 "detail": str(e),
 674             }
 675             return self._format_out(problem_details, None)
 676
 677     @staticmethod
 678     def _format_login(token_info):
 679         """
 680         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 681         log this information
 682         :param token_info: Dictionary with token content
 683         :return: None
 684         """
 685         cherrypy.request.login = token_info.get("username", "-")
 686         if token_info.get("project_name"):
 687             cherrypy.request.login += "/" + token_info["project_name"]
 688         if token_info.get("id"):
 689             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 690
 691     @cherrypy.expose
 692     def token(self, method, token_id=None, kwargs=None):
 693         token_info = None
 694         # self.engine.load_dbase(cherrypy.request.app.config)
 695         indata = self._format_in(kwargs)
 696         if not isinstance(indata, dict):
 697             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 698
 699         if method == "GET":
 700             token_info = self.authenticator.authorize()
 701             # for logging
 702             self._format_login(token_info)
 703             if token_id:
 704                 outdata = self.authenticator.get_token(token_info, token_id)
 705             else:
 706                 outdata = self.authenticator.get_token_list(token_info)
 707         elif method == "POST":
 708             try:
 709                 token_info = self.authenticator.authorize()
 710             except Exception:
 711                 token_info = None
 712             if kwargs:
 713                 indata.update(kwargs)
 714             # This is needed to log the user when authentication fails
 715             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 716             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 717             cherrypy.session['Authorization'] = outdata["_id"]
 718             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 719             # for logging
 720             self._format_login(token_info)
 721
 722             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 723             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 724         elif method == "DELETE":
 725             if not token_id and "id" in kwargs:
 726                 token_id = kwargs["id"]
 727             elif not token_id:
 728                 token_info = self.authenticator.authorize()
 729                 # for logging
 730                 self._format_login(token_info)
 731                 token_id = token_info["_id"]
 732             outdata = self.authenticator.del_token(token_id)
 733             token_info = None
 734             cherrypy.session['Authorization'] = "logout"
 735             # cherrypy.response.cookie["Authorization"] = token_id
 736             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 737         else:
 738             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 739         return self._format_out(outdata, token_info)
 740
 741     @cherrypy.expose
 742     def test(self, *args, **kwargs):
 743         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 744                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 745             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 746             return "test URL is disabled"
 747         thread_info = None
 748         if args and args[0] == "help":
 749             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 750                    "sleep/<time>\nmessage/topic\n</pre></html>"
 751
 752         elif args and args[0] == "init":
 753             try:
 754                 # self.engine.load_dbase(cherrypy.request.app.config)
 755                 self.engine.create_admin()
 756                 return "Done. User 'admin', password 'admin' created"
 757             except Exception:
 758                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 759                 return self._format_out("Database already initialized")
 760         elif args and args[0] == "file":
 761             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 762                                                   "text/plain", "attachment")
 763         elif args and args[0] == "file2":
 764             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 765             f = open(f_path, "r")
 766             cherrypy.response.headers["Content-type"] = "text/plain"
 767             return f
 768
 769         elif len(args) == 2 and args[0] == "db-clear":
 770             deleted_info = self.engine.db.del_list(args[1], kwargs)
 771             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 772         elif len(args) and args[0] == "fs-clear":
 773             if len(args) >= 2:
 774                 folders = (args[1],)
 775             else:
 776                 folders = self.engine.fs.dir_ls(".")
 777             for folder in folders:
 778                 self.engine.fs.file_delete(folder)
 779             return ",".join(folders) + " folders deleted\n"
 780         elif args and args[0] == "login":
 781             if not cherrypy.request.headers.get("Authorization"):
 782                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 783                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 784         elif args and args[0] == "login2":
 785             if not cherrypy.request.headers.get("Authorization"):
 786                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 787                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 788         elif args and args[0] == "sleep":
 789             sleep_time = 5
 790             try:
 791                 sleep_time = int(args[1])
 792             except Exception:
 793                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 794                 return self._format_out("Database already initialized")
 795             thread_info = cherrypy.thread_data
 796             print(thread_info)
 797             time.sleep(sleep_time)
 798             # thread_info
 799         elif len(args) >= 2 and args[0] == "message":
 800             main_topic = args[1]
 801             return_text = "<html><pre>{} ->\n".format(main_topic)
 802             try:
 803                 if cherrypy.request.method == 'POST':
 804                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 805                     for k, v in to_send.items():
 806                         self.engine.msg.write(main_topic, k, v)
 807                         return_text += "  {}: {}\n".format(k, v)
 808                 elif cherrypy.request.method == 'GET':
 809                     for k, v in kwargs.items():
 810                         self.engine.msg.write(main_topic, k, yaml.load(v), Loader=yaml.SafeLoader)
 811                         return_text += "  {}: {}\n".format(k, yaml.load(v), Loader=yaml.SafeLoader)
 812             except Exception as e:
 813                 return_text += "Error: " + str(e)
 814             return_text += "</pre></html>\n"
 815             return return_text
 816
 817         return_text = (
 818             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 819             "  kwargs: {}\n".format(kwargs) +
 820             "  headers: {}\n".format(cherrypy.request.headers) +
 821             "  path_info: {}\n".format(cherrypy.request.path_info) +
 822             "  query_string: {}\n".format(cherrypy.request.query_string) +
 823             "  session: {}\n".format(cherrypy.session) +
 824             "  cookie: {}\n".format(cherrypy.request.cookie) +
 825             "  method: {}\n".format(cherrypy.request.method) +
 826             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 827             "  body:\n")
 828         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 829         if cherrypy.request.body.length:
 830             return_text += "    content: {}\n".format(
 831                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 832         if thread_info:
 833             return_text += "thread: {}\n".format(thread_info)
 834         return_text += "</pre></html>"
 835         return return_text
 836
 837     @staticmethod
 838     def _check_valid_url_method(method, *args):
 839         if len(args) < 3:
 840             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 841
 842         reference = valid_url_methods
 843         for arg in args:
 844             if arg is None:
 845                 break
 846             if not isinstance(reference, dict):
 847                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 848                                    HTTPStatus.METHOD_NOT_ALLOWED)
 849
 850             if arg in reference:
 851                 reference = reference[arg]
 852             elif "<ID>" in reference:
 853                 reference = reference["<ID>"]
 854             elif "*" in reference:
 855                 reference = reference["*"]
 856                 break
 857             else:
 858                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 859         if "TODO" in reference and method in reference["TODO"]:
 860             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 861         elif "METHODS" in reference and method not in reference["METHODS"]:
 862             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 863         return reference["ROLE_PERMISSION"] + method.lower()
 864
 865     @staticmethod
 866     def _set_location_header(main_topic, version, topic, id):
 867         """
 868         Insert response header Location with the URL of created item base on URL params
 869         :param main_topic:
 870         :param version:
 871         :param topic:
 872         :param id:
 873         :return: None
 874         """
 875         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 876         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 877         return
 878
 879     @staticmethod
 880     def _extract_query_string_operations(kwargs, method):
 881         """
 882
 883         :param kwargs:
 884         :return:
 885         """
 886         query_string_operations = []
 887         if kwargs:
 888             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 889                 if qs in kwargs and kwargs[qs].lower() != "false":
 890                     query_string_operations.append(qs.lower() + ":" + method.lower())
 891         return query_string_operations
 892
 893     @staticmethod
 894     def _manage_admin_query(token_info, kwargs, method, _id):
 895         """
 896         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 897         Check that users has rights to use them and returs the admin_query
 898         :param token_info: token_info rights obtained by token
 899         :param kwargs: query string input.
 900         :param method: http method: GET, POSST, PUT, ...
 901         :param _id:
 902         :return: admin_query dictionary with keys:
 903             public: True, False or None
 904             force: True or False
 905             project_id: tuple with projects used for accessing an element
 906             set_project: tuple with projects that a created element will belong to
 907             method: show, list, delete, write
 908         """
 909         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 910                        "admin": token_info["admin"], "public": None,
 911                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 912         if kwargs:
 913             # FORCE
 914             if "FORCE" in kwargs:
 915                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 916                     admin_query["force"] = True
 917                 del kwargs["FORCE"]
 918             # PUBLIC
 919             if "PUBLIC" in kwargs:
 920                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 921                     admin_query["public"] = True
 922                 else:
 923                     admin_query["public"] = False
 924                 del kwargs["PUBLIC"]
 925             # ADMIN
 926             if "ADMIN" in kwargs:
 927                 behave_as = kwargs.pop("ADMIN")
 928                 if behave_as.lower() != "false":
 929                     if not token_info["admin"]:
 930                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 931                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 932                         admin_query["project_id"] = ()
 933                     elif isinstance(behave_as, (list, tuple)):
 934                         admin_query["project_id"] = behave_as
 935                     else:   # isinstance(behave_as, str)
 936                         admin_query["project_id"] = (behave_as, )
 937             if "SET_PROJECT" in kwargs:
 938                 set_project = kwargs.pop("SET_PROJECT")
 939                 if not set_project:
 940                     admin_query["set_project"] = list(admin_query["project_id"])
 941                 else:
 942                     if isinstance(set_project, str):
 943                         set_project = (set_project, )
 944                     if admin_query["project_id"]:
 945                         for p in set_project:
 946                             if p not in admin_query["project_id"]:
 947                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 948                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 949                     admin_query["set_project"] = set_project
 950
 951             # PROJECT_READ
 952             # if "PROJECT_READ" in kwargs:
 953             #     admin_query["project"] = kwargs.pop("project")
 954             #     if admin_query["project"] == token_info["project_id"]:
 955         if method == "GET":
 956             if _id:
 957                 admin_query["method"] = "show"
 958             else:
 959                 admin_query["method"] = "list"
 960         elif method == "DELETE":
 961             admin_query["method"] = "delete"
 962         else:
 963             admin_query["method"] = "write"
 964         return admin_query
 965
 966     @cherrypy.expose
 967     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 968         token_info = None
 969         outdata = None
 970         _format = None
 971         method = "DONE"
 972         engine_topic = None
 973         rollback = []
 974         engine_session = None
 975         try:
 976             if not main_topic or not version or not topic:
 977                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 978                                    HTTPStatus.METHOD_NOT_ALLOWED)
 979             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 980                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 981                                    HTTPStatus.METHOD_NOT_ALLOWED)
 982             if version != 'v1':
 983                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 984
 985             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 986                 method = kwargs.pop("METHOD")
 987             else:
 988                 method = cherrypy.request.method
 989
 990             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 991             query_string_operations = self._extract_query_string_operations(kwargs, method)
 992             if main_topic == "admin" and topic == "tokens":
 993                 return self.token(method, _id, kwargs)
 994             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
 995             if main_topic == "admin" and topic == "domains":
 996                 return self.domain()
 997             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 998             indata = self._format_in(kwargs)
 999             engine_topic = topic
1000             if topic == "subscriptions":
1001                 engine_topic = main_topic + "_" + topic
1002             if item and topic != "pm_jobs":
1003                 engine_topic = item
1004
1005             if main_topic == "nsd":
1006                 engine_topic = "nsds"
1007             elif main_topic == "vnfpkgm":
1008                 engine_topic = "vnfds"
1009                 if topic == "vnfpkg_op_occs":
1010                     engine_topic = "vnfpkgops"
1011                 if topic == "vnf_packages" and item == "action":
1012                     engine_topic = "vnfpkgops"
1013             elif main_topic == "nslcm":
1014                 engine_topic = "nsrs"
1015                 if topic == "ns_lcm_op_occs":
1016                     engine_topic = "nslcmops"
1017                 if topic == "vnfrs" or topic == "vnf_instances":
1018                     engine_topic = "vnfrs"
1019             elif main_topic == "nst":
1020                 engine_topic = "nsts"
1021             elif main_topic == "nsilcm":
1022                 engine_topic = "nsis"
1023                 if topic == "nsi_lcm_op_occs":
1024                     engine_topic = "nsilcmops"
1025             elif main_topic == "pdu":
1026                 engine_topic = "pdus"
1027             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1028                 engine_topic = "vim_accounts"
1029
1030             if method == "GET":
1031                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1032                     if item in ("vnfd", "nsd", "nst"):
1033                         path = "$DESCRIPTOR"
1034                     elif args:
1035                         path = args
1036                     elif item == "artifacts":
1037                         path = ()
1038                     else:
1039                         path = None
1040                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1041                                                          cherrypy.request.headers.get("Accept"))
1042                     outdata = file
1043                 elif not _id:
1044                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1045                 else:
1046                     if item == "reports":
1047                         # TODO check that project_id (_id in this context) has permissions
1048                         _id = args[0]
1049                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1050
1051             elif method == "POST":
1052                 cherrypy.response.status = HTTPStatus.CREATED.value
1053                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1054                     _id = cherrypy.request.headers.get("Transaction-Id")
1055                     if not _id:
1056                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1057                                                       cherrypy.request.headers)
1058                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1059                                                            cherrypy.request.headers)
1060                     if completed:
1061                         self._set_location_header(main_topic, version, topic, _id)
1062                     else:
1063                         cherrypy.response.headers["Transaction-Id"] = _id
1064                     outdata = {"id": _id}
1065                 elif topic == "ns_instances_content":
1066                     # creates NSR
1067                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1068                     # creates nslcmop
1069                     indata["lcmOperationType"] = "instantiate"
1070                     indata["nsInstanceId"] = _id
1071                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1072                     self._set_location_header(main_topic, version, topic, _id)
1073                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1074                 elif topic == "ns_instances" and item:
1075                     indata["lcmOperationType"] = item
1076                     indata["nsInstanceId"] = _id
1077                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1078                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1079                     outdata = {"id": _id}
1080                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1081                 elif topic == "netslice_instances_content":
1082                     # creates NetSlice_Instance_record (NSIR)
1083                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1084                     self._set_location_header(main_topic, version, topic, _id)
1085                     indata["lcmOperationType"] = "instantiate"
1086                     indata["netsliceInstanceId"] = _id
1087                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1088                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1089                 elif topic == "netslice_instances" and item:
1090                     indata["lcmOperationType"] = item
1091                     indata["netsliceInstanceId"] = _id
1092                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1093                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1094                     outdata = {"id": _id}
1095                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1096                 elif topic == "vnf_packages" and item == "action":
1097                     indata["lcmOperationType"] = item
1098                     indata["vnfPkgId"] = _id
1099                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1100                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1101                     outdata = {"id": _id}
1102                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1103                 else:
1104                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1105                                                       cherrypy.request.headers)
1106                     self._set_location_header(main_topic, version, topic, _id)
1107                     outdata = {"id": _id}
1108                     if op_id:
1109                         outdata["op_id"] = op_id
1110                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1111                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1112
1113             elif method == "DELETE":
1114                 if not _id:
1115                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1116                     cherrypy.response.status = HTTPStatus.OK.value
1117                 else:  # len(args) > 1
1118                     # for NS NSI generate an operation
1119                     op_id = None
1120                     if topic == "ns_instances_content" and not engine_session["force"]:
1121                         nslcmop_desc = {
1122                             "lcmOperationType": "terminate",
1123                             "nsInstanceId": _id,
1124                             "autoremove": True
1125                         }
1126                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1127                         if op_id:
1128                             outdata = {"_id": op_id}
1129                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1130                         nsilcmop_desc = {
1131                             "lcmOperationType": "terminate",
1132                             "netsliceInstanceId": _id,
1133                             "autoremove": True
1134                         }
1135                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1136                         if op_id:
1137                             outdata = {"_id": op_id}
1138                     # if there is not any deletion in process, delete
1139                     if not op_id:
1140                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1141                         if op_id:
1142                             outdata = {"op_id": op_id}
1143                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1144
1145             elif method in ("PUT", "PATCH"):
1146                 op_id = None
1147                 if not indata and not kwargs and not engine_session.get("set_project"):
1148                     raise NbiException("Nothing to update. Provide payload and/or query string",
1149                                        HTTPStatus.BAD_REQUEST)
1150                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1151                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1152                                                            cherrypy.request.headers)
1153                     if not completed:
1154                         cherrypy.response.headers["Transaction-Id"] = id
1155                 else:
1156                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1157
1158                 if op_id:
1159                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1160                     outdata = {"op_id": op_id}
1161                 else:
1162                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1163                     outdata = None
1164             else:
1165                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1166
1167             # if Role information changes, it is needed to reload the information of roles
1168             if topic == "roles" and method != "GET":
1169                 self.authenticator.load_operation_to_allowed_roles()
1170
1171             if topic == "projects" and method == "DELETE" \
1172                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1173                 self.authenticator.remove_token_from_cache()
1174
1175             return self._format_out(outdata, token_info, _format)
1176         except Exception as e:
1177             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1178                               ValidationError, AuthconnException)):
1179                 http_code_value = cherrypy.response.status = e.http_code.value
1180                 http_code_name = e.http_code.name
1181                 cherrypy.log("Exception {}".format(e))
1182             else:
1183                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1184                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1185                 http_code_name = HTTPStatus.BAD_REQUEST.name
1186             if hasattr(outdata, "close"):  # is an open file
1187                 outdata.close()
1188             error_text = str(e)
1189             rollback.reverse()
1190             for rollback_item in rollback:
1191                 try:
1192                     if rollback_item.get("operation") == "set":
1193                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1194                                                rollback_item["content"], fail_on_empty=False)
1195                     else:
1196                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1197                                                fail_on_empty=False)
1198                 except Exception as e2:
1199                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1200                     cherrypy.log(rollback_error_text)
1201                     error_text += ". " + rollback_error_text
1202             # if isinstance(e, MsgException):
1203             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1204             #         engine_topic[:-1], method, error_text)
1205             problem_details = {
1206                 "code": http_code_name,
1207                 "status": http_code_value,
1208                 "detail": error_text,
1209             }
1210             return self._format_out(problem_details, token_info)
1211             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1212         finally:
1213             if token_info:
1214                 self._format_login(token_info)
1215                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1216                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1217                         if outdata.get(logging_id):
1218                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1219
1220
1221 def _start_service():
1222     """
1223     Callback function called when cherrypy.engine starts
1224     Override configuration with env variables
1225     Set database, storage, message configuration
1226     Init database with admin/admin user password
1227     """
1228     global nbi_server
1229     global subscription_thread
1230     cherrypy.log.error("Starting osm_nbi")
1231     # update general cherrypy configuration
1232     update_dict = {}
1233
1234     engine_config = cherrypy.tree.apps['/osm'].config
1235     for k, v in environ.items():
1236         if not k.startswith("OSMNBI_"):
1237             continue
1238         k1, _, k2 = k[7:].lower().partition("_")
1239         if not k2:
1240             continue
1241         try:
1242             # update static configuration
1243             if k == 'OSMNBI_STATIC_DIR':
1244                 engine_config["/static"]['tools.staticdir.dir'] = v
1245                 engine_config["/static"]['tools.staticdir.on'] = True
1246             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1247                 update_dict['server.socket_port'] = int(v)
1248             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1249                 update_dict['server.socket_host'] = v
1250             elif k1 in ("server", "test", "auth", "log"):
1251                 update_dict[k1 + '.' + k2] = v
1252             elif k1 in ("message", "database", "storage", "authentication"):
1253                 # k2 = k2.replace('_', '.')
1254                 if k2 in ("port", "db_port"):
1255                     engine_config[k1][k2] = int(v)
1256                 else:
1257                     engine_config[k1][k2] = v
1258
1259         except ValueError as e:
1260             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1261         except Exception as e:
1262             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1263
1264     if update_dict:
1265         cherrypy.config.update(update_dict)
1266         engine_config["global"].update(update_dict)
1267
1268     # logging cherrypy
1269     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1270     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1271     logger_server = logging.getLogger("cherrypy.error")
1272     logger_access = logging.getLogger("cherrypy.access")
1273     logger_cherry = logging.getLogger("cherrypy")
1274     logger_nbi = logging.getLogger("nbi")
1275
1276     if "log.file" in engine_config["global"]:
1277         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1278                                                             maxBytes=100e6, backupCount=9, delay=0)
1279         file_handler.setFormatter(log_formatter_simple)
1280         logger_cherry.addHandler(file_handler)
1281         logger_nbi.addHandler(file_handler)
1282     # log always to standard output
1283     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1284                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1285                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1286                             }.items():
1287         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1288         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1289         str_handler = logging.StreamHandler()
1290         str_handler.setFormatter(log_formatter_cherry)
1291         logger.addHandler(str_handler)
1292
1293     if engine_config["global"].get("log.level"):
1294         logger_cherry.setLevel(engine_config["global"]["log.level"])
1295         logger_nbi.setLevel(engine_config["global"]["log.level"])
1296
1297     # logging other modules
1298     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1299         engine_config[k1]["logger_name"] = logname
1300         logger_module = logging.getLogger(logname)
1301         if "logfile" in engine_config[k1]:
1302             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1303                                                                 maxBytes=100e6, backupCount=9, delay=0)
1304             file_handler.setFormatter(log_formatter_simple)
1305             logger_module.addHandler(file_handler)
1306         if "loglevel" in engine_config[k1]:
1307             logger_module.setLevel(engine_config[k1]["loglevel"])
1308     # TODO add more entries, e.g.: storage
1309     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1310     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1311     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1312     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1313
1314     # start subscriptions thread:
1315     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1316     subscription_thread.start()
1317     # Do not capture except SubscriptionException
1318
1319     backend = engine_config["authentication"]["backend"]
1320     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1321                        .format(nbi_version, nbi_version_date, backend))
1322
1323
1324 def _stop_service():
1325     """
1326     Callback function called when cherrypy.engine stops
1327     TODO: Ending database connections.
1328     """
1329     global subscription_thread
1330     if subscription_thread:
1331         subscription_thread.terminate()
1332     subscription_thread = None
1333     cherrypy.tree.apps['/osm'].root.engine.stop()
1334     cherrypy.log.error("Stopping osm_nbi")
1335
1336
1337 def nbi(config_file):
1338     global nbi_server
1339     # conf = {
1340     #     '/': {
1341     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1342     #         'tools.sessions.on': True,
1343     #         'tools.response_headers.on': True,
1344     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1345     #     }
1346     # }
1347     # cherrypy.Server.ssl_module = 'builtin'
1348     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1349     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1350     # cherrypy.Server.thread_pool = 10
1351     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1352
1353     # cherrypy.config.update({'tools.auth_basic.on': True,
1354     #    'tools.auth_basic.realm': 'localhost',
1355     #    'tools.auth_basic.checkpassword': validate_password})
1356     nbi_server = Server()
1357     cherrypy.engine.subscribe('start', _start_service)
1358     cherrypy.engine.subscribe('stop', _stop_service)
1359     cherrypy.quickstart(nbi_server, '/osm', config_file)
1360
1361
1362 def usage():
1363     print("""Usage: {} [options]
1364         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1365         -h|--help: shows this help
1366         """.format(sys.argv[0]))
1367     # --log-socket-host HOST: send logs to this host")
1368     # --log-socket-port PORT: send logs using this port (default: 9022)")
1369
1370
1371 if __name__ == '__main__':
1372     try:
1373         # load parameters and configuration
1374         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1375         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1376         config_file = None
1377         for o, a in opts:
1378             if o in ("-h", "--help"):
1379                 usage()
1380                 sys.exit()
1381             elif o in ("-c", "--config"):
1382                 config_file = a
1383             # elif o == "--log-socket-port":
1384             #     log_socket_port = a
1385             # elif o == "--log-socket-host":
1386             #     log_socket_host = a
1387             # elif o == "--log-file":
1388             #     log_file = a
1389             else:
1390                 assert False, "Unhandled option"
1391         if config_file:
1392             if not path.isfile(config_file):
1393                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1394                 exit(1)
1395         else:
1396             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1397                 if path.isfile(config_file):
1398                     break
1399             else:
1400                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1401                 exit(1)
1402         nbi(config_file)
1403     except getopt.GetoptError as e:
1404         print(str(e), file=sys.stderr)
1405         # usage()
1406         exit(1)