osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118             /osmrepos                                           O       O
 119                 /<id>                                           O                               O
 120
 121         /nst/v1                                                 O       O
 122             /netslice_templates_content                         O       O
 123                 /<nstInfoId>                                    O       O       O       O
 124             /netslice_templates                                 O       O
 125                 /<nstInfoId>                                    O                       O       O
 126                     /nst_content                                O               O
 127                     /nst                                        O
 128                     /artifacts[/<artifactPath>]                 O
 129             /subscriptions                                      X       X
 130                 /<subscriptionId>                               X                       X
 131
 132         /nsilcm/v1
 133             /netslice_instances_content                         O       O
 134                 /<SliceInstanceId>                              O                       O
 135             /netslice_instances                                 O       O
 136                 /<SliceInstanceId>                              O                       O
 137                     instantiate                                         O
 138                     terminate                                           O
 139                     action                                              O
 140             /nsi_lcm_op_occs                                    O       O
 141                 /<nsiLcmOpOccId>                                O                       O       O
 142             /subscriptions                                      X       X
 143                 /<subscriptionId>                               X                       X
 144
 145 query string:
 146     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 147         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 148         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 149         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 150         attrName := string
 151     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 152     item of the array, that is, pass if any item of the array pass the filter.
 153     It allows both ne and neq for not equal
 154     TODO: 4.3.3 Attribute selectors
 155         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 156         (none)  … same as “exclude_default”
 157         all_fields      … all attributes.
 158         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 159         conditionally mandatory, and that are not provided in <list>.
 160         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 161         are not conditionally mandatory, and that are provided in <list>.
 162         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 163         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 164         the particular resource
 165         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 166         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 167         present specification for the particular resource, but that are not part of <list>
 168     Additionally it admits some administrator values:
 169         FORCE: To force operations skipping dependency checkings
 170         ADMIN: To act as an administrator or a different project
 171         PUBLIC: To get public descriptors or set a descriptor as public
 172         SET_PROJECT: To make a descriptor available for other project
 173
 174 Header field name       Reference       Example Descriptions
 175     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 176     This header field shall be present if the response is expected to have a non-empty message body.
 177     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 178     This header field shall be present if the request has a non-empty message body.
 179     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 180     Details are specified in clause 4.5.3.
 181     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 182 Header field name       Reference       Example Descriptions
 183     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 184     This header field shall be present if the response has a non-empty message body.
 185     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 186     new resource has been created.
 187     This header field shall be present if the response status code is 201 or 3xx.
 188     In the present document this header field is also used if the response status code is 202 and a new resource was
 189     created.
 190     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 191     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 192     token.
 193     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 194     certain resources.
 195     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 196     response, and the total length of the file.
 197     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 198 """
 199
 200 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 201 # ^ Contains possible administrative query string words:
 202 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 203 #           (not owned by my session project).
 204 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 205 #     FORCE=True(by default)|False: Force edition/deletion operations
 206 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 207
 208 valid_url_methods = {
 209     # contains allowed URL and methods, and the role_permission name
 210     "admin": {
 211         "v1": {
 212             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 213                        "ROLE_PERMISSION": "tokens:",
 214                        "<ID>": {"METHODS": ("GET", "DELETE"),
 215                                 "ROLE_PERMISSION": "tokens:id:"
 216                                 }
 217                        },
 218             "users": {"METHODS": ("GET", "POST"),
 219                       "ROLE_PERMISSION": "users:",
 220                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 221                                "ROLE_PERMISSION": "users:id:"
 222                                }
 223                       },
 224             "projects": {"METHODS": ("GET", "POST"),
 225                          "ROLE_PERMISSION": "projects:",
 226                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 227                                   "ROLE_PERMISSION": "projects:id:"}
 228                          },
 229             "roles": {"METHODS": ("GET", "POST"),
 230                       "ROLE_PERMISSION": "roles:",
 231                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 232                                "ROLE_PERMISSION": "roles:id:"
 233                                }
 234                       },
 235             "vims": {"METHODS": ("GET", "POST"),
 236                      "ROLE_PERMISSION": "vims:",
 237                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 238                               "ROLE_PERMISSION": "vims:id:"
 239                               }
 240                      },
 241             "vim_accounts": {"METHODS": ("GET", "POST"),
 242                              "ROLE_PERMISSION": "vim_accounts:",
 243                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 244                                       "ROLE_PERMISSION": "vim_accounts:id:"
 245                                       }
 246                              },
 247             "wim_accounts": {"METHODS": ("GET", "POST"),
 248                              "ROLE_PERMISSION": "wim_accounts:",
 249                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 250                                       "ROLE_PERMISSION": "wim_accounts:id:"
 251                                       }
 252                              },
 253             "sdns": {"METHODS": ("GET", "POST"),
 254                      "ROLE_PERMISSION": "sdn_controllers:",
 255                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 256                               "ROLE_PERMISSION": "sdn_controllers:id:"
 257                               }
 258                      },
 259             "k8sclusters": {"METHODS": ("GET", "POST"),
 260                             "ROLE_PERMISSION": "k8sclusters:",
 261                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 262                                      "ROLE_PERMISSION": "k8sclusters:id:"
 263                                      }
 264                             },
 265             "vca": {"METHODS": ("GET", "POST"),
 266                     "ROLE_PERMISSION": "vca:",
 267                     "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 268                              "ROLE_PERMISSION": "vca:id:"
 269                              }
 270                     },
 271             "k8srepos": {"METHODS": ("GET", "POST"),
 272                          "ROLE_PERMISSION": "k8srepos:",
 273                          "<ID>": {"METHODS": ("GET", "DELETE"),
 274                                   "ROLE_PERMISSION": "k8srepos:id:"
 275                                   }
 276                          },
 277             "osmrepos": {"METHODS": ("GET", "POST"),
 278                          "ROLE_PERMISSION": "osmrepos:",
 279                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 280                                   "ROLE_PERMISSION": "osmrepos:id:"
 281                                   }
 282                          },
 283             "domains": {"METHODS": ("GET", ),
 284                         "ROLE_PERMISSION": "domains:",
 285                         },
 286         }
 287     },
 288     "pdu": {
 289         "v1": {
 290             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 291                                 "ROLE_PERMISSION": "pduds:",
 292                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 293                                          "ROLE_PERMISSION": "pduds:id:"
 294                                          }
 295                                 },
 296         }
 297     },
 298     "nsd": {
 299         "v1": {
 300             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 301                                        "ROLE_PERMISSION": "nsds:",
 302                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 303                                                 "ROLE_PERMISSION": "nsds:id:"
 304                                                 }
 305                                        },
 306             "ns_descriptors": {"METHODS": ("GET", "POST"),
 307                                "ROLE_PERMISSION": "nsds:",
 308                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 309                                         "ROLE_PERMISSION": "nsds:id:",
 310                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 311                                                         "ROLE_PERMISSION": "nsds:id:content:",
 312                                                         },
 313                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 314                                                 "ROLE_PERMISSION": "nsds:id:content:"
 315                                                 },
 316                                         "artifacts": {"METHODS": ("GET",),
 317                                                       "ROLE_PERMISSION": "nsds:id:nsd_artifact:",
 318                                                       "*": None,
 319                                                       }
 320                                         }
 321                                },
 322             "pnf_descriptors": {"TODO": ("GET", "POST"),
 323                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 324                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 325                                          }
 326                                 },
 327             "subscriptions": {"TODO": ("GET", "POST"),
 328                               "<ID>": {"TODO": ("GET", "DELETE")}
 329                               },
 330         }
 331     },
 332     "vnfpkgm": {
 333         "v1": {
 334             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 335                                      "ROLE_PERMISSION": "vnfds:",
 336                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 337                                               "ROLE_PERMISSION": "vnfds:id:"}
 338                                      },
 339             "vnf_packages": {"METHODS": ("GET", "POST"),
 340                              "ROLE_PERMISSION": "vnfds:",
 341                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 342                                       "ROLE_PERMISSION": "vnfds:id:",
 343                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 344                                                           "ROLE_PERMISSION": "vnfds:id:",
 345                                                           "upload_from_uri": {"METHODS": (),
 346                                                                               "TODO": ("POST", ),
 347                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 348                                                                               }
 349                                                           },
 350                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 351                                                "ROLE_PERMISSION": "vnfds:id:content:"
 352                                                },
 353                                       "artifacts": {"METHODS": ("GET", ),
 354                                                     "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:",
 355                                                     "*": None,
 356                                                     },
 357                                       "action": {"METHODS": ("POST", ),
 358                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 359                                                  },
 360                                       }
 361                              },
 362             "subscriptions": {"TODO": ("GET", "POST"),
 363                               "<ID>": {"TODO": ("GET", "DELETE")}
 364                               },
 365             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 366                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 367                                "<ID>": {"METHODS": ("GET", ),
 368                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 369                                         }
 370                                },
 371         }
 372     },
 373     "nslcm": {
 374         "v1": {
 375             "ns_instances_content": {"METHODS": ("GET", "POST"),
 376                                      "ROLE_PERMISSION": "ns_instances:",
 377                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 378                                               "ROLE_PERMISSION": "ns_instances:id:"
 379                                               }
 380                                      },
 381             "ns_instances": {"METHODS": ("GET", "POST"),
 382                              "ROLE_PERMISSION": "ns_instances:",
 383                              "<ID>": {"METHODS": ("GET", "DELETE"),
 384                                       "ROLE_PERMISSION": "ns_instances:id:",
 385                                       "scale": {"METHODS": ("POST",),
 386                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 387                                                 },
 388                                       "terminate": {"METHODS": ("POST",),
 389                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 390                                                     },
 391                                       "instantiate": {"METHODS": ("POST",),
 392                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 393                                                       },
 394                                       "action": {"METHODS": ("POST",),
 395                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 396                                                  },
 397                                       }
 398                              },
 399             "ns_lcm_op_occs": {"METHODS": ("GET",),
 400                                "ROLE_PERMISSION": "ns_instances:opps:",
 401                                "<ID>": {"METHODS": ("GET",),
 402                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 403                                         },
 404                                },
 405             "vnfrs": {"METHODS": ("GET",),
 406                       "ROLE_PERMISSION": "vnf_instances:",
 407                       "<ID>": {"METHODS": ("GET",),
 408                                "ROLE_PERMISSION": "vnf_instances:id:"
 409                                }
 410                       },
 411             "vnf_instances": {"METHODS": ("GET",),
 412                               "ROLE_PERMISSION": "vnf_instances:",
 413                               "<ID>": {"METHODS": ("GET",),
 414                                        "ROLE_PERMISSION": "vnf_instances:id:"
 415                                        }
 416                               },
 417             "subscriptions": {"METHODS": ("GET", "POST"),
 418                               "ROLE_PERMISSION": "ns_subscriptions:",
 419                               "<ID>": {"METHODS": ("GET", "DELETE"),
 420                                        "ROLE_PERMISSION": "ns_subscriptions:id:"
 421                                        }
 422                               },
 423         }
 424     },
 425     "nst": {
 426         "v1": {
 427             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 428                                            "ROLE_PERMISSION": "slice_templates:",
 429                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 430                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 431                                            },
 432             "netslice_templates": {"METHODS": ("GET", "POST"),
 433                                    "ROLE_PERMISSION": "slice_templates:",
 434                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 435                                             "TODO": ("PATCH",),
 436                                             "ROLE_PERMISSION": "slice_templates:id:",
 437                                             "nst_content": {"METHODS": ("GET", "PUT"),
 438                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 439                                                             },
 440                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 441                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 442                                                     },
 443                                             "artifacts": {"METHODS": ("GET",),
 444                                                           "ROLE_PERMISSION": "slice_templates:id:content:",
 445                                                           "*": None
 446                                                           }
 447                                             }
 448                                    },
 449             "subscriptions": {"TODO": ("GET", "POST"),
 450                               "<ID>": {"TODO": ("GET", "DELETE")}
 451                               },
 452         }
 453     },
 454     "nsilcm": {
 455         "v1": {
 456             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 457                                            "ROLE_PERMISSION": "slice_instances:",
 458                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 459                                                     "ROLE_PERMISSION": "slice_instances:id:"
 460                                                     }
 461                                            },
 462             "netslice_instances": {"METHODS": ("GET", "POST"),
 463                                    "ROLE_PERMISSION": "slice_instances:",
 464                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 465                                             "ROLE_PERMISSION": "slice_instances:id:",
 466                                             "terminate": {"METHODS": ("POST",),
 467                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 468                                                           },
 469                                             "instantiate": {"METHODS": ("POST",),
 470                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 471                                                             },
 472                                             "action": {"METHODS": ("POST",),
 473                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 474                                                        },
 475                                             }
 476                                    },
 477             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 478                                 "ROLE_PERMISSION": "slice_instances:opps:",
 479                                 "<ID>": {"METHODS": ("GET",),
 480                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 481                                          },
 482                                 },
 483         }
 484     },
 485     "nspm": {
 486         "v1": {
 487             "pm_jobs": {
 488                 "<ID>": {
 489                     "reports": {
 490                         "<ID>": {"METHODS": ("GET",),
 491                                  "ROLE_PERMISSION": "reports:id:",
 492                                  }
 493                     }
 494                 },
 495             },
 496         },
 497     },
 498 }
 499
 500
 501 class NbiException(Exception):
 502
 503     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 504         Exception.__init__(self, message)
 505         self.http_code = http_code
 506
 507
 508 class Server(object):
 509     instance = 0
 510     # to decode bytes to str
 511     reader = getreader("utf-8")
 512
 513     def __init__(self):
 514         self.instance += 1
 515         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 516         self.engine = Engine(self.authenticator)
 517
 518     def _format_in(self, kwargs):
 519         try:
 520             indata = None
 521             if cherrypy.request.body.length:
 522                 error_text = "Invalid input format "
 523
 524                 if "Content-Type" in cherrypy.request.headers:
 525                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 526                         error_text = "Invalid json format "
 527                         indata = json.load(self.reader(cherrypy.request.body))
 528                         cherrypy.request.headers.pop("Content-File-MD5", None)
 529                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 530                         error_text = "Invalid yaml format "
 531                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 532                         cherrypy.request.headers.pop("Content-File-MD5", None)
 533                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 534                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 535                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 536                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 537                         indata = cherrypy.request.body  # .read()
 538                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 539                         if "descriptor_file" in kwargs:
 540                             filecontent = kwargs.pop("descriptor_file")
 541                             if not filecontent.file:
 542                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 543                             indata = filecontent.file  # .read()
 544                             if filecontent.content_type.value:
 545                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 546                     else:
 547                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 548                         #                          "Only 'Content-Type' of type 'application/json' or
 549                         # 'application/yaml' for input format are available")
 550                         error_text = "Invalid yaml format "
 551                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 552                         cherrypy.request.headers.pop("Content-File-MD5", None)
 553                 else:
 554                     error_text = "Invalid yaml format "
 555                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 556                     cherrypy.request.headers.pop("Content-File-MD5", None)
 557             if not indata:
 558                 indata = {}
 559
 560             format_yaml = False
 561             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 562                 format_yaml = True
 563
 564             for k, v in kwargs.items():
 565                 if isinstance(v, str):
 566                     if v == "":
 567                         kwargs[k] = None
 568                     elif format_yaml:
 569                         try:
 570                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 571                         except Exception:
 572                             pass
 573                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 574                         try:
 575                             kwargs[k] = int(v)
 576                         except Exception:
 577                             try:
 578                                 kwargs[k] = float(v)
 579                             except Exception:
 580                                 pass
 581                     elif v.find(",") > 0:
 582                         kwargs[k] = v.split(",")
 583                 elif isinstance(v, (list, tuple)):
 584                     for index in range(0, len(v)):
 585                         if v[index] == "":
 586                             v[index] = None
 587                         elif format_yaml:
 588                             try:
 589                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 590                             except Exception:
 591                                 pass
 592
 593             return indata
 594         except (ValueError, yaml.YAMLError) as exc:
 595             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 596         except KeyError as exc:
 597             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 598         except Exception as exc:
 599             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 600
 601     @staticmethod
 602     def _format_out(data, token_info=None, _format=None):
 603         """
 604         return string of dictionary data according to requested json, yaml, xml. By default json
 605         :param data: response to be sent. Can be a dict, text or file
 606         :param token_info: Contains among other username and project
 607         :param _format: The format to be set as Content-Type if data is a file
 608         :return: None
 609         """
 610         accept = cherrypy.request.headers.get("Accept")
 611         if data is None:
 612             if accept and "text/html" in accept:
 613                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 614             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 615             return
 616         elif hasattr(data, "read"):  # file object
 617             if _format:
 618                 cherrypy.response.headers["Content-Type"] = _format
 619             elif "b" in data.mode:  # binariy asssumig zip
 620                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 621             else:
 622                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 623             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 624             return data
 625         if accept:
 626             if "text/html" in accept:
 627                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 628             elif "application/yaml" in accept or "*/*" in accept:
 629                 pass
 630             elif "application/json" in accept or (cherrypy.response.status and cherrypy.response.status >= 300):
 631                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 632                 a = json.dumps(data, indent=4) + "\n"
 633                 return a.encode("utf8")
 634         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 635         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 636                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 637
 638     @cherrypy.expose
 639     def index(self, *args, **kwargs):
 640         token_info = None
 641         try:
 642             if cherrypy.request.method == "GET":
 643                 token_info = self.authenticator.authorize()
 644                 outdata = token_info   # Home page
 645             else:
 646                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 647                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 648
 649             return self._format_out(outdata, token_info)
 650
 651         except (EngineException, AuthException) as e:
 652             # cherrypy.log("index Exception {}".format(e))
 653             cherrypy.response.status = e.http_code.value
 654             return self._format_out("Welcome to OSM!", token_info)
 655
 656     @cherrypy.expose
 657     def version(self, *args, **kwargs):
 658         # TODO consider to remove and provide version using the static version file
 659         try:
 660             if cherrypy.request.method != "GET":
 661                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 662             elif args or kwargs:
 663                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 664             # TODO include version of other modules, pick up from some kafka admin message
 665             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 666             return self._format_out(osm_nbi_version)
 667         except NbiException as e:
 668             cherrypy.response.status = e.http_code.value
 669             problem_details = {
 670                 "code": e.http_code.name,
 671                 "status": e.http_code.value,
 672                 "detail": str(e),
 673             }
 674             return self._format_out(problem_details, None)
 675
 676     def domain(self):
 677         try:
 678             domains = {
 679                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 680                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 681             return self._format_out(domains)
 682         except NbiException as e:
 683             cherrypy.response.status = e.http_code.value
 684             problem_details = {
 685                 "code": e.http_code.name,
 686                 "status": e.http_code.value,
 687                 "detail": str(e),
 688             }
 689             return self._format_out(problem_details, None)
 690
 691     @staticmethod
 692     def _format_login(token_info):
 693         """
 694         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 695         log this information
 696         :param token_info: Dictionary with token content
 697         :return: None
 698         """
 699         cherrypy.request.login = token_info.get("username", "-")
 700         if token_info.get("project_name"):
 701             cherrypy.request.login += "/" + token_info["project_name"]
 702         if token_info.get("id"):
 703             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 704
 705     @cherrypy.expose
 706     def token(self, method, token_id=None, kwargs=None):
 707         token_info = None
 708         # self.engine.load_dbase(cherrypy.request.app.config)
 709         indata = self._format_in(kwargs)
 710         if not isinstance(indata, dict):
 711             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 712
 713         if method == "GET":
 714             token_info = self.authenticator.authorize()
 715             # for logging
 716             self._format_login(token_info)
 717             if token_id:
 718                 outdata = self.authenticator.get_token(token_info, token_id)
 719             else:
 720                 outdata = self.authenticator.get_token_list(token_info)
 721         elif method == "POST":
 722             try:
 723                 token_info = self.authenticator.authorize()
 724             except Exception:
 725                 token_info = None
 726             if kwargs:
 727                 indata.update(kwargs)
 728             # This is needed to log the user when authentication fails
 729             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 730             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 731             cherrypy.session['Authorization'] = outdata["_id"]
 732             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 733             # for logging
 734             self._format_login(token_info)
 735
 736             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 737             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 738         elif method == "DELETE":
 739             if not token_id and "id" in kwargs:
 740                 token_id = kwargs["id"]
 741             elif not token_id:
 742                 token_info = self.authenticator.authorize()
 743                 # for logging
 744                 self._format_login(token_info)
 745                 token_id = token_info["_id"]
 746             outdata = self.authenticator.del_token(token_id)
 747             token_info = None
 748             cherrypy.session['Authorization'] = "logout"
 749             # cherrypy.response.cookie["Authorization"] = token_id
 750             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 751         else:
 752             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 753         return self._format_out(outdata, token_info)
 754
 755     @cherrypy.expose
 756     def test(self, *args, **kwargs):
 757         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 758                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 759             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 760             return "test URL is disabled"
 761         thread_info = None
 762         if args and args[0] == "help":
 763             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 764                    "sleep/<time>\nmessage/topic\n</pre></html>"
 765
 766         elif args and args[0] == "init":
 767             try:
 768                 # self.engine.load_dbase(cherrypy.request.app.config)
 769                 self.engine.create_admin()
 770                 return "Done. User 'admin', password 'admin' created"
 771             except Exception:
 772                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 773                 return self._format_out("Database already initialized")
 774         elif args and args[0] == "file":
 775             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 776                                                   "text/plain", "attachment")
 777         elif args and args[0] == "file2":
 778             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 779             f = open(f_path, "r")
 780             cherrypy.response.headers["Content-type"] = "text/plain"
 781             return f
 782
 783         elif len(args) == 2 and args[0] == "db-clear":
 784             deleted_info = self.engine.db.del_list(args[1], kwargs)
 785             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 786         elif len(args) and args[0] == "fs-clear":
 787             if len(args) >= 2:
 788                 folders = (args[1],)
 789             else:
 790                 folders = self.engine.fs.dir_ls(".")
 791             for folder in folders:
 792                 self.engine.fs.file_delete(folder)
 793             return ",".join(folders) + " folders deleted\n"
 794         elif args and args[0] == "login":
 795             if not cherrypy.request.headers.get("Authorization"):
 796                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 797                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 798         elif args and args[0] == "login2":
 799             if not cherrypy.request.headers.get("Authorization"):
 800                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 801                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 802         elif args and args[0] == "sleep":
 803             sleep_time = 5
 804             try:
 805                 sleep_time = int(args[1])
 806             except Exception:
 807                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 808                 return self._format_out("Database already initialized")
 809             thread_info = cherrypy.thread_data
 810             print(thread_info)
 811             time.sleep(sleep_time)
 812             # thread_info
 813         elif len(args) >= 2 and args[0] == "message":
 814             main_topic = args[1]
 815             return_text = "<html><pre>{} ->\n".format(main_topic)
 816             try:
 817                 if cherrypy.request.method == 'POST':
 818                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 819                     for k, v in to_send.items():
 820                         self.engine.msg.write(main_topic, k, v)
 821                         return_text += "  {}: {}\n".format(k, v)
 822                 elif cherrypy.request.method == 'GET':
 823                     for k, v in kwargs.items():
 824                         v_dict = yaml.load(v, Loader=yaml.SafeLoader)
 825                         self.engine.msg.write(main_topic, k, v_dict)
 826                         return_text += "  {}: {}\n".format(k, v_dict)
 827             except Exception as e:
 828                 return_text += "Error: " + str(e)
 829             return_text += "</pre></html>\n"
 830             return return_text
 831
 832         return_text = (
 833             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 834             "  kwargs: {}\n".format(kwargs) +
 835             "  headers: {}\n".format(cherrypy.request.headers) +
 836             "  path_info: {}\n".format(cherrypy.request.path_info) +
 837             "  query_string: {}\n".format(cherrypy.request.query_string) +
 838             "  session: {}\n".format(cherrypy.session) +
 839             "  cookie: {}\n".format(cherrypy.request.cookie) +
 840             "  method: {}\n".format(cherrypy.request.method) +
 841             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 842             "  body:\n")
 843         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 844         if cherrypy.request.body.length:
 845             return_text += "    content: {}\n".format(
 846                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 847         if thread_info:
 848             return_text += "thread: {}\n".format(thread_info)
 849         return_text += "</pre></html>"
 850         return return_text
 851
 852     @staticmethod
 853     def _check_valid_url_method(method, *args):
 854         if len(args) < 3:
 855             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 856
 857         reference = valid_url_methods
 858         for arg in args:
 859             if arg is None:
 860                 break
 861             if not isinstance(reference, dict):
 862                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 863                                    HTTPStatus.METHOD_NOT_ALLOWED)
 864
 865             if arg in reference:
 866                 reference = reference[arg]
 867             elif "<ID>" in reference:
 868                 reference = reference["<ID>"]
 869             elif "*" in reference:
 870                 # if there is content
 871                 if reference["*"]:
 872                     reference = reference["*"]
 873                 break
 874             else:
 875                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 876         if "TODO" in reference and method in reference["TODO"]:
 877             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 878         elif "METHODS" in reference and method not in reference["METHODS"]:
 879             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 880         return reference["ROLE_PERMISSION"] + method.lower()
 881
 882     @staticmethod
 883     def _set_location_header(main_topic, version, topic, id):
 884         """
 885         Insert response header Location with the URL of created item base on URL params
 886         :param main_topic:
 887         :param version:
 888         :param topic:
 889         :param id:
 890         :return: None
 891         """
 892         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 893         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 894         return
 895
 896     @staticmethod
 897     def _extract_query_string_operations(kwargs, method):
 898         """
 899
 900         :param kwargs:
 901         :return:
 902         """
 903         query_string_operations = []
 904         if kwargs:
 905             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 906                 if qs in kwargs and kwargs[qs].lower() != "false":
 907                     query_string_operations.append(qs.lower() + ":" + method.lower())
 908         return query_string_operations
 909
 910     @staticmethod
 911     def _manage_admin_query(token_info, kwargs, method, _id):
 912         """
 913         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 914         Check that users has rights to use them and returs the admin_query
 915         :param token_info: token_info rights obtained by token
 916         :param kwargs: query string input.
 917         :param method: http method: GET, POSST, PUT, ...
 918         :param _id:
 919         :return: admin_query dictionary with keys:
 920             public: True, False or None
 921             force: True or False
 922             project_id: tuple with projects used for accessing an element
 923             set_project: tuple with projects that a created element will belong to
 924             method: show, list, delete, write
 925         """
 926         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 927                        "admin": token_info["admin"], "public": None,
 928                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 929         if kwargs:
 930             # FORCE
 931             if "FORCE" in kwargs:
 932                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 933                     admin_query["force"] = True
 934                 del kwargs["FORCE"]
 935             # PUBLIC
 936             if "PUBLIC" in kwargs:
 937                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 938                     admin_query["public"] = True
 939                 else:
 940                     admin_query["public"] = False
 941                 del kwargs["PUBLIC"]
 942             # ADMIN
 943             if "ADMIN" in kwargs:
 944                 behave_as = kwargs.pop("ADMIN")
 945                 if behave_as.lower() != "false":
 946                     if not token_info["admin"]:
 947                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 948                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 949                         admin_query["project_id"] = ()
 950                     elif isinstance(behave_as, (list, tuple)):
 951                         admin_query["project_id"] = behave_as
 952                     else:   # isinstance(behave_as, str)
 953                         admin_query["project_id"] = (behave_as, )
 954             if "SET_PROJECT" in kwargs:
 955                 set_project = kwargs.pop("SET_PROJECT")
 956                 if not set_project:
 957                     admin_query["set_project"] = list(admin_query["project_id"])
 958                 else:
 959                     if isinstance(set_project, str):
 960                         set_project = (set_project, )
 961                     if admin_query["project_id"]:
 962                         for p in set_project:
 963                             if p not in admin_query["project_id"]:
 964                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 965                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 966                     admin_query["set_project"] = set_project
 967
 968             # PROJECT_READ
 969             # if "PROJECT_READ" in kwargs:
 970             #     admin_query["project"] = kwargs.pop("project")
 971             #     if admin_query["project"] == token_info["project_id"]:
 972         if method == "GET":
 973             if _id:
 974                 admin_query["method"] = "show"
 975             else:
 976                 admin_query["method"] = "list"
 977         elif method == "DELETE":
 978             admin_query["method"] = "delete"
 979         else:
 980             admin_query["method"] = "write"
 981         return admin_query
 982
 983     @cherrypy.expose
 984     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 985         token_info = None
 986         outdata = None
 987         _format = None
 988         method = "DONE"
 989         engine_topic = None
 990         rollback = []
 991         engine_session = None
 992         try:
 993             if not main_topic or not version or not topic:
 994                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 995                                    HTTPStatus.METHOD_NOT_ALLOWED)
 996             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 997                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 998                                    HTTPStatus.METHOD_NOT_ALLOWED)
 999             if version != 'v1':
1000                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
1001
1002             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
1003                 method = kwargs.pop("METHOD")
1004             else:
1005                 method = cherrypy.request.method
1006
1007             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
1008             query_string_operations = self._extract_query_string_operations(kwargs, method)
1009             if main_topic == "admin" and topic == "tokens":
1010                 return self.token(method, _id, kwargs)
1011             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
1012             if main_topic == "admin" and topic == "domains":
1013                 return self.domain()
1014             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
1015             indata = self._format_in(kwargs)
1016             engine_topic = topic
1017
1018             if item and topic != "pm_jobs":
1019                 engine_topic = item
1020
1021             if main_topic == "nsd":
1022                 engine_topic = "nsds"
1023             elif main_topic == "vnfpkgm":
1024                 engine_topic = "vnfds"
1025                 if topic == "vnfpkg_op_occs":
1026                     engine_topic = "vnfpkgops"
1027                 if topic == "vnf_packages" and item == "action":
1028                     engine_topic = "vnfpkgops"
1029             elif main_topic == "nslcm":
1030                 engine_topic = "nsrs"
1031                 if topic == "ns_lcm_op_occs":
1032                     engine_topic = "nslcmops"
1033                 if topic == "vnfrs" or topic == "vnf_instances":
1034                     engine_topic = "vnfrs"
1035             elif main_topic == "nst":
1036                 engine_topic = "nsts"
1037             elif main_topic == "nsilcm":
1038                 engine_topic = "nsis"
1039                 if topic == "nsi_lcm_op_occs":
1040                     engine_topic = "nsilcmops"
1041             elif main_topic == "pdu":
1042                 engine_topic = "pdus"
1043             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1044                 engine_topic = "vim_accounts"
1045
1046             if topic == "subscriptions":
1047                 engine_topic = main_topic + "_" + topic
1048
1049             if method == "GET":
1050                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1051                     if item in ("vnfd", "nsd", "nst"):
1052                         path = "$DESCRIPTOR"
1053                     elif args:
1054                         path = args
1055                     elif item == "artifacts":
1056                         path = ()
1057                     else:
1058                         path = None
1059                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1060                                                          cherrypy.request.headers.get("Accept"))
1061                     outdata = file
1062                 elif not _id:
1063                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs, api_req=True)
1064                 else:
1065                     if item == "reports":
1066                         # TODO check that project_id (_id in this context) has permissions
1067                         _id = args[0]
1068                     outdata = self.engine.get_item(engine_session, engine_topic, _id, True)
1069
1070             elif method == "POST":
1071                 cherrypy.response.status = HTTPStatus.CREATED.value
1072                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1073                     _id = cherrypy.request.headers.get("Transaction-Id")
1074                     if not _id:
1075                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1076                                                       cherrypy.request.headers)
1077                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1078                                                            cherrypy.request.headers)
1079                     if completed:
1080                         self._set_location_header(main_topic, version, topic, _id)
1081                     else:
1082                         cherrypy.response.headers["Transaction-Id"] = _id
1083                     outdata = {"id": _id}
1084                 elif topic == "ns_instances_content":
1085                     # creates NSR
1086                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1087                     # creates nslcmop
1088                     indata["lcmOperationType"] = "instantiate"
1089                     indata["nsInstanceId"] = _id
1090                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1091                     self._set_location_header(main_topic, version, topic, _id)
1092                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1093                 elif topic == "ns_instances" and item:
1094                     indata["lcmOperationType"] = item
1095                     indata["nsInstanceId"] = _id
1096                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1097                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1098                     outdata = {"id": _id}
1099                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1100                 elif topic == "netslice_instances_content":
1101                     # creates NetSlice_Instance_record (NSIR)
1102                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1103                     self._set_location_header(main_topic, version, topic, _id)
1104                     indata["lcmOperationType"] = "instantiate"
1105                     indata["netsliceInstanceId"] = _id
1106                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1107                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1108                 elif topic == "netslice_instances" and item:
1109                     indata["lcmOperationType"] = item
1110                     indata["netsliceInstanceId"] = _id
1111                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1112                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1113                     outdata = {"id": _id}
1114                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1115                 elif topic == "vnf_packages" and item == "action":
1116                     indata["lcmOperationType"] = item
1117                     indata["vnfPkgId"] = _id
1118                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1119                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1120                     outdata = {"id": _id}
1121                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1122                 elif topic == "subscriptions":
1123                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1124                     self._set_location_header(main_topic, version, topic, _id)
1125                     link = {}
1126                     link["self"] = cherrypy.response.headers["Location"]
1127                     outdata = {"id": _id, "filter": indata["filter"], "callbackUri": indata["CallbackUri"],
1128                                "_links": link}
1129                     cherrypy.response.status = HTTPStatus.CREATED.value
1130                 else:
1131                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1132                                                       cherrypy.request.headers)
1133                     self._set_location_header(main_topic, version, topic, _id)
1134                     outdata = {"id": _id}
1135                     if op_id:
1136                         outdata["op_id"] = op_id
1137                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1138                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1139
1140             elif method == "DELETE":
1141                 if not _id:
1142                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1143                     cherrypy.response.status = HTTPStatus.OK.value
1144                 else:  # len(args) > 1
1145                     # for NS NSI generate an operation
1146                     op_id = None
1147                     if topic == "ns_instances_content" and not engine_session["force"]:
1148                         nslcmop_desc = {
1149                             "lcmOperationType": "terminate",
1150                             "nsInstanceId": _id,
1151                             "autoremove": True
1152                         }
1153                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1154                         if op_id:
1155                             outdata = {"_id": op_id}
1156                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1157                         nsilcmop_desc = {
1158                             "lcmOperationType": "terminate",
1159                             "netsliceInstanceId": _id,
1160                             "autoremove": True
1161                         }
1162                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1163                         if op_id:
1164                             outdata = {"_id": op_id}
1165                     # if there is not any deletion in process, delete
1166                     if not op_id:
1167                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1168                         if op_id:
1169                             outdata = {"op_id": op_id}
1170                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1171
1172             elif method in ("PUT", "PATCH"):
1173                 op_id = None
1174                 if not indata and not kwargs and not engine_session.get("set_project"):
1175                     raise NbiException("Nothing to update. Provide payload and/or query string",
1176                                        HTTPStatus.BAD_REQUEST)
1177                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1178                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1179                                                            cherrypy.request.headers)
1180                     if not completed:
1181                         cherrypy.response.headers["Transaction-Id"] = id
1182                 else:
1183                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1184
1185                 if op_id:
1186                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1187                     outdata = {"op_id": op_id}
1188                 else:
1189                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1190                     outdata = None
1191             else:
1192                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1193
1194             # if Role information changes, it is needed to reload the information of roles
1195             if topic == "roles" and method != "GET":
1196                 self.authenticator.load_operation_to_allowed_roles()
1197
1198             if topic == "projects" and method == "DELETE" \
1199                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1200                 self.authenticator.remove_token_from_cache()
1201
1202             return self._format_out(outdata, token_info, _format)
1203         except Exception as e:
1204             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1205                               ValidationError, AuthconnException)):
1206                 http_code_value = cherrypy.response.status = e.http_code.value
1207                 http_code_name = e.http_code.name
1208                 cherrypy.log("Exception {}".format(e))
1209             else:
1210                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1211                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1212                 http_code_name = HTTPStatus.BAD_REQUEST.name
1213             if hasattr(outdata, "close"):  # is an open file
1214                 outdata.close()
1215             error_text = str(e)
1216             rollback.reverse()
1217             for rollback_item in rollback:
1218                 try:
1219                     if rollback_item.get("operation") == "set":
1220                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1221                                                rollback_item["content"], fail_on_empty=False)
1222                     elif rollback_item.get("operation") == "del_list":
1223                         self.engine.db.del_list(rollback_item["topic"], rollback_item["filter"],
1224                                                 fail_on_empty=False)
1225                     else:
1226                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1227                                                fail_on_empty=False)
1228                 except Exception as e2:
1229                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1230                     cherrypy.log(rollback_error_text)
1231                     error_text += ". " + rollback_error_text
1232             # if isinstance(e, MsgException):
1233             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1234             #         engine_topic[:-1], method, error_text)
1235             problem_details = {
1236                 "code": http_code_name,
1237                 "status": http_code_value,
1238                 "detail": error_text,
1239             }
1240             return self._format_out(problem_details, token_info)
1241             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1242         finally:
1243             if token_info:
1244                 self._format_login(token_info)
1245                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1246                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1247                         if outdata.get(logging_id):
1248                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1249
1250
1251 def _start_service():
1252     """
1253     Callback function called when cherrypy.engine starts
1254     Override configuration with env variables
1255     Set database, storage, message configuration
1256     Init database with admin/admin user password
1257     """
1258     global nbi_server
1259     global subscription_thread
1260     cherrypy.log.error("Starting osm_nbi")
1261     # update general cherrypy configuration
1262     update_dict = {}
1263
1264     engine_config = cherrypy.tree.apps['/osm'].config
1265     for k, v in environ.items():
1266         if not k.startswith("OSMNBI_"):
1267             continue
1268         k1, _, k2 = k[7:].lower().partition("_")
1269         if not k2:
1270             continue
1271         try:
1272             # update static configuration
1273             if k == 'OSMNBI_STATIC_DIR':
1274                 engine_config["/static"]['tools.staticdir.dir'] = v
1275                 engine_config["/static"]['tools.staticdir.on'] = True
1276             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1277                 update_dict['server.socket_port'] = int(v)
1278             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1279                 update_dict['server.socket_host'] = v
1280             elif k1 in ("server", "test", "auth", "log"):
1281                 update_dict[k1 + '.' + k2] = v
1282             elif k1 in ("message", "database", "storage", "authentication"):
1283                 # k2 = k2.replace('_', '.')
1284                 if k2 in ("port", "db_port"):
1285                     engine_config[k1][k2] = int(v)
1286                 else:
1287                     engine_config[k1][k2] = v
1288
1289         except ValueError as e:
1290             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1291         except Exception as e:
1292             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1293
1294     if update_dict:
1295         cherrypy.config.update(update_dict)
1296         engine_config["global"].update(update_dict)
1297
1298     # logging cherrypy
1299     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1300     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1301     logger_server = logging.getLogger("cherrypy.error")
1302     logger_access = logging.getLogger("cherrypy.access")
1303     logger_cherry = logging.getLogger("cherrypy")
1304     logger_nbi = logging.getLogger("nbi")
1305
1306     if "log.file" in engine_config["global"]:
1307         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1308                                                             maxBytes=100e6, backupCount=9, delay=0)
1309         file_handler.setFormatter(log_formatter_simple)
1310         logger_cherry.addHandler(file_handler)
1311         logger_nbi.addHandler(file_handler)
1312     # log always to standard output
1313     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1314                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1315                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1316                             }.items():
1317         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1318         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1319         str_handler = logging.StreamHandler()
1320         str_handler.setFormatter(log_formatter_cherry)
1321         logger.addHandler(str_handler)
1322
1323     if engine_config["global"].get("log.level"):
1324         logger_cherry.setLevel(engine_config["global"]["log.level"])
1325         logger_nbi.setLevel(engine_config["global"]["log.level"])
1326
1327     # logging other modules
1328     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1329         engine_config[k1]["logger_name"] = logname
1330         logger_module = logging.getLogger(logname)
1331         if "logfile" in engine_config[k1]:
1332             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1333                                                                 maxBytes=100e6, backupCount=9, delay=0)
1334             file_handler.setFormatter(log_formatter_simple)
1335             logger_module.addHandler(file_handler)
1336         if "loglevel" in engine_config[k1]:
1337             logger_module.setLevel(engine_config[k1]["loglevel"])
1338     # TODO add more entries, e.g.: storage
1339     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1340     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1341     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1342     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1343
1344     # start subscriptions thread:
1345     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1346     subscription_thread.start()
1347     # Do not capture except SubscriptionException
1348
1349     backend = engine_config["authentication"]["backend"]
1350     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1351                        .format(nbi_version, nbi_version_date, backend))
1352
1353
1354 def _stop_service():
1355     """
1356     Callback function called when cherrypy.engine stops
1357     TODO: Ending database connections.
1358     """
1359     global subscription_thread
1360     if subscription_thread:
1361         subscription_thread.terminate()
1362     subscription_thread = None
1363     cherrypy.tree.apps['/osm'].root.engine.stop()
1364     cherrypy.log.error("Stopping osm_nbi")
1365
1366
1367 def nbi(config_file):
1368     global nbi_server
1369     # conf = {
1370     #     '/': {
1371     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1372     #         'tools.sessions.on': True,
1373     #         'tools.response_headers.on': True,
1374     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1375     #     }
1376     # }
1377     # cherrypy.Server.ssl_module = 'builtin'
1378     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1379     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1380     # cherrypy.Server.thread_pool = 10
1381     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1382
1383     # cherrypy.config.update({'tools.auth_basic.on': True,
1384     #    'tools.auth_basic.realm': 'localhost',
1385     #    'tools.auth_basic.checkpassword': validate_password})
1386     nbi_server = Server()
1387     cherrypy.engine.subscribe('start', _start_service)
1388     cherrypy.engine.subscribe('stop', _stop_service)
1389     cherrypy.quickstart(nbi_server, '/osm', config_file)
1390
1391
1392 def usage():
1393     print("""Usage: {} [options]
1394         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1395         -h|--help: shows this help
1396         """.format(sys.argv[0]))
1397     # --log-socket-host HOST: send logs to this host")
1398     # --log-socket-port PORT: send logs using this port (default: 9022)")
1399
1400
1401 if __name__ == '__main__':
1402     try:
1403         # load parameters and configuration
1404         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1405         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1406         config_file = None
1407         for o, a in opts:
1408             if o in ("-h", "--help"):
1409                 usage()
1410                 sys.exit()
1411             elif o in ("-c", "--config"):
1412                 config_file = a
1413             # elif o == "--log-socket-port":
1414             #     log_socket_port = a
1415             # elif o == "--log-socket-host":
1416             #     log_socket_host = a
1417             # elif o == "--log-file":
1418             #     log_file = a
1419             else:
1420                 assert False, "Unhandled option"
1421         if config_file:
1422             if not path.isfile(config_file):
1423                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1424                 exit(1)
1425         else:
1426             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1427                 if path.isfile(config_file):
1428                     break
1429             else:
1430                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1431                 exit(1)
1432         nbi(config_file)
1433     except getopt.GetoptError as e:
1434         print(str(e), file=sys.stderr)
1435         # usage()
1436         exit(1)