osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118             /osmrepos                                           O       O
 119                 /<id>                                           O                               O
 120
 121         /nst/v1                                                 O       O
 122             /netslice_templates_content                         O       O
 123                 /<nstInfoId>                                    O       O       O       O
 124             /netslice_templates                                 O       O
 125                 /<nstInfoId>                                    O                       O       O
 126                     /nst_content                                O               O
 127                     /nst                                        O
 128                     /artifacts[/<artifactPath>]                 O
 129             /subscriptions                                      X       X
 130                 /<subscriptionId>                               X                       X
 131
 132         /nsilcm/v1
 133             /netslice_instances_content                         O       O
 134                 /<SliceInstanceId>                              O                       O
 135             /netslice_instances                                 O       O
 136                 /<SliceInstanceId>                              O                       O
 137                     instantiate                                         O
 138                     terminate                                           O
 139                     action                                              O
 140             /nsi_lcm_op_occs                                    O       O
 141                 /<nsiLcmOpOccId>                                O                       O       O
 142             /subscriptions                                      X       X
 143                 /<subscriptionId>                               X                       X
 144
 145 query string:
 146     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 147         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 148         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 149         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 150         attrName := string
 151     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 152     item of the array, that is, pass if any item of the array pass the filter.
 153     It allows both ne and neq for not equal
 154     TODO: 4.3.3 Attribute selectors
 155         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 156         (none)  … same as “exclude_default”
 157         all_fields      … all attributes.
 158         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 159         conditionally mandatory, and that are not provided in <list>.
 160         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 161         are not conditionally mandatory, and that are provided in <list>.
 162         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 163         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 164         the particular resource
 165         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 166         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 167         present specification for the particular resource, but that are not part of <list>
 168     Additionally it admits some administrator values:
 169         FORCE: To force operations skipping dependency checkings
 170         ADMIN: To act as an administrator or a different project
 171         PUBLIC: To get public descriptors or set a descriptor as public
 172         SET_PROJECT: To make a descriptor available for other project
 173
 174 Header field name       Reference       Example Descriptions
 175     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 176     This header field shall be present if the response is expected to have a non-empty message body.
 177     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 178     This header field shall be present if the request has a non-empty message body.
 179     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 180     Details are specified in clause 4.5.3.
 181     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 182 Header field name       Reference       Example Descriptions
 183     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 184     This header field shall be present if the response has a non-empty message body.
 185     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 186     new resource has been created.
 187     This header field shall be present if the response status code is 201 or 3xx.
 188     In the present document this header field is also used if the response status code is 202 and a new resource was
 189     created.
 190     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 191     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 192     token.
 193     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 194     certain resources.
 195     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 196     response, and the total length of the file.
 197     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 198 """
 199
 200 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 201 # ^ Contains possible administrative query string words:
 202 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 203 #           (not owned by my session project).
 204 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 205 #     FORCE=True(by default)|False: Force edition/deletion operations
 206 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 207
 208 valid_url_methods = {
 209     # contains allowed URL and methods, and the role_permission name
 210     "admin": {
 211         "v1": {
 212             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 213                        "ROLE_PERMISSION": "tokens:",
 214                        "<ID>": {"METHODS": ("GET", "DELETE"),
 215                                 "ROLE_PERMISSION": "tokens:id:"
 216                                 }
 217                        },
 218             "users": {"METHODS": ("GET", "POST"),
 219                       "ROLE_PERMISSION": "users:",
 220                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 221                                "ROLE_PERMISSION": "users:id:"
 222                                }
 223                       },
 224             "projects": {"METHODS": ("GET", "POST"),
 225                          "ROLE_PERMISSION": "projects:",
 226                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 227                                   "ROLE_PERMISSION": "projects:id:"}
 228                          },
 229             "roles": {"METHODS": ("GET", "POST"),
 230                       "ROLE_PERMISSION": "roles:",
 231                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 232                                "ROLE_PERMISSION": "roles:id:"
 233                                }
 234                       },
 235             "vims": {"METHODS": ("GET", "POST"),
 236                      "ROLE_PERMISSION": "vims:",
 237                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 238                               "ROLE_PERMISSION": "vims:id:"
 239                               }
 240                      },
 241             "vim_accounts": {"METHODS": ("GET", "POST"),
 242                              "ROLE_PERMISSION": "vim_accounts:",
 243                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 244                                       "ROLE_PERMISSION": "vim_accounts:id:"
 245                                       }
 246                              },
 247             "wim_accounts": {"METHODS": ("GET", "POST"),
 248                              "ROLE_PERMISSION": "wim_accounts:",
 249                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 250                                       "ROLE_PERMISSION": "wim_accounts:id:"
 251                                       }
 252                              },
 253             "sdns": {"METHODS": ("GET", "POST"),
 254                      "ROLE_PERMISSION": "sdn_controllers:",
 255                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 256                               "ROLE_PERMISSION": "sdn_controllers:id:"
 257                               }
 258                      },
 259             "k8sclusters": {"METHODS": ("GET", "POST"),
 260                             "ROLE_PERMISSION": "k8sclusters:",
 261                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 262                                      "ROLE_PERMISSION": "k8sclusters:id:"
 263                                      }
 264                             },
 265             "k8srepos": {"METHODS": ("GET", "POST"),
 266                          "ROLE_PERMISSION": "k8srepos:",
 267                          "<ID>": {"METHODS": ("GET", "DELETE"),
 268                                   "ROLE_PERMISSION": "k8srepos:id:"
 269                                   }
 270                          },
 271             "osmrepos": {"METHODS": ("GET", "POST"),
 272                          "ROLE_PERMISSION": "osmrepos:",
 273                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 274                                   "ROLE_PERMISSION": "osmrepos:id:"
 275                                   }
 276                          },
 277             "domains": {"METHODS": ("GET", ),
 278                         "ROLE_PERMISSION": "domains:",
 279                         },
 280         }
 281     },
 282     "pdu": {
 283         "v1": {
 284             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 285                                 "ROLE_PERMISSION": "pduds:",
 286                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 287                                          "ROLE_PERMISSION": "pduds:id:"
 288                                          }
 289                                 },
 290         }
 291     },
 292     "nsd": {
 293         "v1": {
 294             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 295                                        "ROLE_PERMISSION": "nsds:",
 296                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 297                                                 "ROLE_PERMISSION": "nsds:id:"
 298                                                 }
 299                                        },
 300             "ns_descriptors": {"METHODS": ("GET", "POST"),
 301                                "ROLE_PERMISSION": "nsds:",
 302                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 303                                         "ROLE_PERMISSION": "nsds:id:",
 304                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 305                                                         "ROLE_PERMISSION": "nsds:id:content:",
 306                                                         },
 307                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 308                                                 "ROLE_PERMISSION": "nsds:id:content:"
 309                                                 },
 310                                         "artifacts": {"METHODS": ("GET",),
 311                                                       "ROLE_PERMISSION": "nsds:id:nsd_artifact:",
 312                                                       "*": None,
 313                                                       }
 314                                         }
 315                                },
 316             "pnf_descriptors": {"TODO": ("GET", "POST"),
 317                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 318                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 319                                          }
 320                                 },
 321             "subscriptions": {"TODO": ("GET", "POST"),
 322                               "<ID>": {"TODO": ("GET", "DELETE")}
 323                               },
 324         }
 325     },
 326     "vnfpkgm": {
 327         "v1": {
 328             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 329                                      "ROLE_PERMISSION": "vnfds:",
 330                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 331                                               "ROLE_PERMISSION": "vnfds:id:"}
 332                                      },
 333             "vnf_packages": {"METHODS": ("GET", "POST"),
 334                              "ROLE_PERMISSION": "vnfds:",
 335                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 336                                       "ROLE_PERMISSION": "vnfds:id:",
 337                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 338                                                           "ROLE_PERMISSION": "vnfds:id:",
 339                                                           "upload_from_uri": {"METHODS": (),
 340                                                                               "TODO": ("POST", ),
 341                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 342                                                                               }
 343                                                           },
 344                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 345                                                "ROLE_PERMISSION": "vnfds:id:content:"
 346                                                },
 347                                       "artifacts": {"METHODS": ("GET", ),
 348                                                     "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:",
 349                                                     "*": None,
 350                                                     },
 351                                       "action": {"METHODS": ("POST", ),
 352                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 353                                                  },
 354                                       }
 355                              },
 356             "subscriptions": {"TODO": ("GET", "POST"),
 357                               "<ID>": {"TODO": ("GET", "DELETE")}
 358                               },
 359             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 360                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 361                                "<ID>": {"METHODS": ("GET", ),
 362                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 363                                         }
 364                                },
 365         }
 366     },
 367     "nslcm": {
 368         "v1": {
 369             "ns_instances_content": {"METHODS": ("GET", "POST"),
 370                                      "ROLE_PERMISSION": "ns_instances:",
 371                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 372                                               "ROLE_PERMISSION": "ns_instances:id:"
 373                                               }
 374                                      },
 375             "ns_instances": {"METHODS": ("GET", "POST"),
 376                              "ROLE_PERMISSION": "ns_instances:",
 377                              "<ID>": {"METHODS": ("GET", "DELETE"),
 378                                       "ROLE_PERMISSION": "ns_instances:id:",
 379                                       "scale": {"METHODS": ("POST",),
 380                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 381                                                 },
 382                                       "terminate": {"METHODS": ("POST",),
 383                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 384                                                     },
 385                                       "instantiate": {"METHODS": ("POST",),
 386                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 387                                                       },
 388                                       "action": {"METHODS": ("POST",),
 389                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 390                                                  },
 391                                       }
 392                              },
 393             "ns_lcm_op_occs": {"METHODS": ("GET",),
 394                                "ROLE_PERMISSION": "ns_instances:opps:",
 395                                "<ID>": {"METHODS": ("GET",),
 396                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 397                                         },
 398                                },
 399             "vnfrs": {"METHODS": ("GET",),
 400                       "ROLE_PERMISSION": "vnf_instances:",
 401                       "<ID>": {"METHODS": ("GET",),
 402                                "ROLE_PERMISSION": "vnf_instances:id:"
 403                                }
 404                       },
 405             "vnf_instances": {"METHODS": ("GET",),
 406                               "ROLE_PERMISSION": "vnf_instances:",
 407                               "<ID>": {"METHODS": ("GET",),
 408                                        "ROLE_PERMISSION": "vnf_instances:id:"
 409                                        }
 410                               },
 411             "subscriptions": {"METHODS": ("GET", "POST"),
 412                               "ROLE_PERMISSION": "ns_subscriptions:",
 413                               "<ID>": {"METHODS": ("GET", "DELETE"),
 414                                        "ROLE_PERMISSION": "ns_subscriptions:id:"
 415                                        }
 416                               },
 417         }
 418     },
 419     "nst": {
 420         "v1": {
 421             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 422                                            "ROLE_PERMISSION": "slice_templates:",
 423                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 424                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 425                                            },
 426             "netslice_templates": {"METHODS": ("GET", "POST"),
 427                                    "ROLE_PERMISSION": "slice_templates:",
 428                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 429                                             "TODO": ("PATCH",),
 430                                             "ROLE_PERMISSION": "slice_templates:id:",
 431                                             "nst_content": {"METHODS": ("GET", "PUT"),
 432                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 433                                                             },
 434                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 435                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 436                                                     },
 437                                             "artifacts": {"METHODS": ("GET",),
 438                                                           "ROLE_PERMISSION": "slice_templates:id:content:",
 439                                                           "*": None
 440                                                           }
 441                                             }
 442                                    },
 443             "subscriptions": {"TODO": ("GET", "POST"),
 444                               "<ID>": {"TODO": ("GET", "DELETE")}
 445                               },
 446         }
 447     },
 448     "nsilcm": {
 449         "v1": {
 450             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 451                                            "ROLE_PERMISSION": "slice_instances:",
 452                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 453                                                     "ROLE_PERMISSION": "slice_instances:id:"
 454                                                     }
 455                                            },
 456             "netslice_instances": {"METHODS": ("GET", "POST"),
 457                                    "ROLE_PERMISSION": "slice_instances:",
 458                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 459                                             "ROLE_PERMISSION": "slice_instances:id:",
 460                                             "terminate": {"METHODS": ("POST",),
 461                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 462                                                           },
 463                                             "instantiate": {"METHODS": ("POST",),
 464                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 465                                                             },
 466                                             "action": {"METHODS": ("POST",),
 467                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 468                                                        },
 469                                             }
 470                                    },
 471             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 472                                 "ROLE_PERMISSION": "slice_instances:opps:",
 473                                 "<ID>": {"METHODS": ("GET",),
 474                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 475                                          },
 476                                 },
 477         }
 478     },
 479     "nspm": {
 480         "v1": {
 481             "pm_jobs": {
 482                 "<ID>": {
 483                     "reports": {
 484                         "<ID>": {"METHODS": ("GET",),
 485                                  "ROLE_PERMISSION": "reports:id:",
 486                                  }
 487                     }
 488                 },
 489             },
 490         },
 491     },
 492 }
 493
 494
 495 class NbiException(Exception):
 496
 497     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 498         Exception.__init__(self, message)
 499         self.http_code = http_code
 500
 501
 502 class Server(object):
 503     instance = 0
 504     # to decode bytes to str
 505     reader = getreader("utf-8")
 506
 507     def __init__(self):
 508         self.instance += 1
 509         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 510         self.engine = Engine(self.authenticator)
 511
 512     def _format_in(self, kwargs):
 513         try:
 514             indata = None
 515             if cherrypy.request.body.length:
 516                 error_text = "Invalid input format "
 517
 518                 if "Content-Type" in cherrypy.request.headers:
 519                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 520                         error_text = "Invalid json format "
 521                         indata = json.load(self.reader(cherrypy.request.body))
 522                         cherrypy.request.headers.pop("Content-File-MD5", None)
 523                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 524                         error_text = "Invalid yaml format "
 525                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 526                         cherrypy.request.headers.pop("Content-File-MD5", None)
 527                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 528                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 529                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 530                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 531                         indata = cherrypy.request.body  # .read()
 532                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 533                         if "descriptor_file" in kwargs:
 534                             filecontent = kwargs.pop("descriptor_file")
 535                             if not filecontent.file:
 536                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 537                             indata = filecontent.file  # .read()
 538                             if filecontent.content_type.value:
 539                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 540                     else:
 541                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 542                         #                          "Only 'Content-Type' of type 'application/json' or
 543                         # 'application/yaml' for input format are available")
 544                         error_text = "Invalid yaml format "
 545                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 546                         cherrypy.request.headers.pop("Content-File-MD5", None)
 547                 else:
 548                     error_text = "Invalid yaml format "
 549                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 550                     cherrypy.request.headers.pop("Content-File-MD5", None)
 551             if not indata:
 552                 indata = {}
 553
 554             format_yaml = False
 555             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 556                 format_yaml = True
 557
 558             for k, v in kwargs.items():
 559                 if isinstance(v, str):
 560                     if v == "":
 561                         kwargs[k] = None
 562                     elif format_yaml:
 563                         try:
 564                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 565                         except Exception:
 566                             pass
 567                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 568                         try:
 569                             kwargs[k] = int(v)
 570                         except Exception:
 571                             try:
 572                                 kwargs[k] = float(v)
 573                             except Exception:
 574                                 pass
 575                     elif v.find(",") > 0:
 576                         kwargs[k] = v.split(",")
 577                 elif isinstance(v, (list, tuple)):
 578                     for index in range(0, len(v)):
 579                         if v[index] == "":
 580                             v[index] = None
 581                         elif format_yaml:
 582                             try:
 583                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 584                             except Exception:
 585                                 pass
 586
 587             return indata
 588         except (ValueError, yaml.YAMLError) as exc:
 589             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 590         except KeyError as exc:
 591             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 592         except Exception as exc:
 593             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 594
 595     @staticmethod
 596     def _format_out(data, token_info=None, _format=None):
 597         """
 598         return string of dictionary data according to requested json, yaml, xml. By default json
 599         :param data: response to be sent. Can be a dict, text or file
 600         :param token_info: Contains among other username and project
 601         :param _format: The format to be set as Content-Type if data is a file
 602         :return: None
 603         """
 604         accept = cherrypy.request.headers.get("Accept")
 605         if data is None:
 606             if accept and "text/html" in accept:
 607                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 608             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 609             return
 610         elif hasattr(data, "read"):  # file object
 611             if _format:
 612                 cherrypy.response.headers["Content-Type"] = _format
 613             elif "b" in data.mode:  # binariy asssumig zip
 614                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 615             else:
 616                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 617             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 618             return data
 619         if accept:
 620             if "text/html" in accept:
 621                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 622             elif "application/yaml" in accept or "*/*" in accept:
 623                 pass
 624             elif "application/json" in accept or (cherrypy.response.status and cherrypy.response.status >= 300):
 625                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 626                 a = json.dumps(data, indent=4) + "\n"
 627                 return a.encode("utf8")
 628         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 629         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 630                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 631
 632     @cherrypy.expose
 633     def index(self, *args, **kwargs):
 634         token_info = None
 635         try:
 636             if cherrypy.request.method == "GET":
 637                 token_info = self.authenticator.authorize()
 638                 outdata = token_info   # Home page
 639             else:
 640                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 641                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 642
 643             return self._format_out(outdata, token_info)
 644
 645         except (EngineException, AuthException) as e:
 646             # cherrypy.log("index Exception {}".format(e))
 647             cherrypy.response.status = e.http_code.value
 648             return self._format_out("Welcome to OSM!", token_info)
 649
 650     @cherrypy.expose
 651     def version(self, *args, **kwargs):
 652         # TODO consider to remove and provide version using the static version file
 653         try:
 654             if cherrypy.request.method != "GET":
 655                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 656             elif args or kwargs:
 657                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 658             # TODO include version of other modules, pick up from some kafka admin message
 659             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 660             return self._format_out(osm_nbi_version)
 661         except NbiException as e:
 662             cherrypy.response.status = e.http_code.value
 663             problem_details = {
 664                 "code": e.http_code.name,
 665                 "status": e.http_code.value,
 666                 "detail": str(e),
 667             }
 668             return self._format_out(problem_details, None)
 669
 670     def domain(self):
 671         try:
 672             domains = {
 673                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 674                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 675             return self._format_out(domains)
 676         except NbiException as e:
 677             cherrypy.response.status = e.http_code.value
 678             problem_details = {
 679                 "code": e.http_code.name,
 680                 "status": e.http_code.value,
 681                 "detail": str(e),
 682             }
 683             return self._format_out(problem_details, None)
 684
 685     @staticmethod
 686     def _format_login(token_info):
 687         """
 688         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 689         log this information
 690         :param token_info: Dictionary with token content
 691         :return: None
 692         """
 693         cherrypy.request.login = token_info.get("username", "-")
 694         if token_info.get("project_name"):
 695             cherrypy.request.login += "/" + token_info["project_name"]
 696         if token_info.get("id"):
 697             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 698
 699     @cherrypy.expose
 700     def token(self, method, token_id=None, kwargs=None):
 701         token_info = None
 702         # self.engine.load_dbase(cherrypy.request.app.config)
 703         indata = self._format_in(kwargs)
 704         if not isinstance(indata, dict):
 705             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 706
 707         if method == "GET":
 708             token_info = self.authenticator.authorize()
 709             # for logging
 710             self._format_login(token_info)
 711             if token_id:
 712                 outdata = self.authenticator.get_token(token_info, token_id)
 713             else:
 714                 outdata = self.authenticator.get_token_list(token_info)
 715         elif method == "POST":
 716             try:
 717                 token_info = self.authenticator.authorize()
 718             except Exception:
 719                 token_info = None
 720             if kwargs:
 721                 indata.update(kwargs)
 722             # This is needed to log the user when authentication fails
 723             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 724             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 725             cherrypy.session['Authorization'] = outdata["_id"]
 726             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 727             # for logging
 728             self._format_login(token_info)
 729
 730             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 731             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 732         elif method == "DELETE":
 733             if not token_id and "id" in kwargs:
 734                 token_id = kwargs["id"]
 735             elif not token_id:
 736                 token_info = self.authenticator.authorize()
 737                 # for logging
 738                 self._format_login(token_info)
 739                 token_id = token_info["_id"]
 740             outdata = self.authenticator.del_token(token_id)
 741             token_info = None
 742             cherrypy.session['Authorization'] = "logout"
 743             # cherrypy.response.cookie["Authorization"] = token_id
 744             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 745         else:
 746             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 747         return self._format_out(outdata, token_info)
 748
 749     @cherrypy.expose
 750     def test(self, *args, **kwargs):
 751         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 752                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 753             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 754             return "test URL is disabled"
 755         thread_info = None
 756         if args and args[0] == "help":
 757             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 758                    "sleep/<time>\nmessage/topic\n</pre></html>"
 759
 760         elif args and args[0] == "init":
 761             try:
 762                 # self.engine.load_dbase(cherrypy.request.app.config)
 763                 self.engine.create_admin()
 764                 return "Done. User 'admin', password 'admin' created"
 765             except Exception:
 766                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 767                 return self._format_out("Database already initialized")
 768         elif args and args[0] == "file":
 769             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 770                                                   "text/plain", "attachment")
 771         elif args and args[0] == "file2":
 772             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 773             f = open(f_path, "r")
 774             cherrypy.response.headers["Content-type"] = "text/plain"
 775             return f
 776
 777         elif len(args) == 2 and args[0] == "db-clear":
 778             deleted_info = self.engine.db.del_list(args[1], kwargs)
 779             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 780         elif len(args) and args[0] == "fs-clear":
 781             if len(args) >= 2:
 782                 folders = (args[1],)
 783             else:
 784                 folders = self.engine.fs.dir_ls(".")
 785             for folder in folders:
 786                 self.engine.fs.file_delete(folder)
 787             return ",".join(folders) + " folders deleted\n"
 788         elif args and args[0] == "login":
 789             if not cherrypy.request.headers.get("Authorization"):
 790                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 791                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 792         elif args and args[0] == "login2":
 793             if not cherrypy.request.headers.get("Authorization"):
 794                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 795                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 796         elif args and args[0] == "sleep":
 797             sleep_time = 5
 798             try:
 799                 sleep_time = int(args[1])
 800             except Exception:
 801                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 802                 return self._format_out("Database already initialized")
 803             thread_info = cherrypy.thread_data
 804             print(thread_info)
 805             time.sleep(sleep_time)
 806             # thread_info
 807         elif len(args) >= 2 and args[0] == "message":
 808             main_topic = args[1]
 809             return_text = "<html><pre>{} ->\n".format(main_topic)
 810             try:
 811                 if cherrypy.request.method == 'POST':
 812                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 813                     for k, v in to_send.items():
 814                         self.engine.msg.write(main_topic, k, v)
 815                         return_text += "  {}: {}\n".format(k, v)
 816                 elif cherrypy.request.method == 'GET':
 817                     for k, v in kwargs.items():
 818                         v_dict = yaml.load(v, Loader=yaml.SafeLoader)
 819                         self.engine.msg.write(main_topic, k, v_dict)
 820                         return_text += "  {}: {}\n".format(k, v_dict)
 821             except Exception as e:
 822                 return_text += "Error: " + str(e)
 823             return_text += "</pre></html>\n"
 824             return return_text
 825
 826         return_text = (
 827             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 828             "  kwargs: {}\n".format(kwargs) +
 829             "  headers: {}\n".format(cherrypy.request.headers) +
 830             "  path_info: {}\n".format(cherrypy.request.path_info) +
 831             "  query_string: {}\n".format(cherrypy.request.query_string) +
 832             "  session: {}\n".format(cherrypy.session) +
 833             "  cookie: {}\n".format(cherrypy.request.cookie) +
 834             "  method: {}\n".format(cherrypy.request.method) +
 835             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 836             "  body:\n")
 837         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 838         if cherrypy.request.body.length:
 839             return_text += "    content: {}\n".format(
 840                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 841         if thread_info:
 842             return_text += "thread: {}\n".format(thread_info)
 843         return_text += "</pre></html>"
 844         return return_text
 845
 846     @staticmethod
 847     def _check_valid_url_method(method, *args):
 848         if len(args) < 3:
 849             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 850
 851         reference = valid_url_methods
 852         for arg in args:
 853             if arg is None:
 854                 break
 855             if not isinstance(reference, dict):
 856                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 857                                    HTTPStatus.METHOD_NOT_ALLOWED)
 858
 859             if arg in reference:
 860                 reference = reference[arg]
 861             elif "<ID>" in reference:
 862                 reference = reference["<ID>"]
 863             elif "*" in reference:
 864                 # if there is content
 865                 if reference["*"]:
 866                     reference = reference["*"]
 867                 break
 868             else:
 869                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 870         if "TODO" in reference and method in reference["TODO"]:
 871             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 872         elif "METHODS" in reference and method not in reference["METHODS"]:
 873             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 874         return reference["ROLE_PERMISSION"] + method.lower()
 875
 876     @staticmethod
 877     def _set_location_header(main_topic, version, topic, id):
 878         """
 879         Insert response header Location with the URL of created item base on URL params
 880         :param main_topic:
 881         :param version:
 882         :param topic:
 883         :param id:
 884         :return: None
 885         """
 886         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 887         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 888         return
 889
 890     @staticmethod
 891     def _extract_query_string_operations(kwargs, method):
 892         """
 893
 894         :param kwargs:
 895         :return:
 896         """
 897         query_string_operations = []
 898         if kwargs:
 899             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 900                 if qs in kwargs and kwargs[qs].lower() != "false":
 901                     query_string_operations.append(qs.lower() + ":" + method.lower())
 902         return query_string_operations
 903
 904     @staticmethod
 905     def _manage_admin_query(token_info, kwargs, method, _id):
 906         """
 907         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 908         Check that users has rights to use them and returs the admin_query
 909         :param token_info: token_info rights obtained by token
 910         :param kwargs: query string input.
 911         :param method: http method: GET, POSST, PUT, ...
 912         :param _id:
 913         :return: admin_query dictionary with keys:
 914             public: True, False or None
 915             force: True or False
 916             project_id: tuple with projects used for accessing an element
 917             set_project: tuple with projects that a created element will belong to
 918             method: show, list, delete, write
 919         """
 920         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 921                        "admin": token_info["admin"], "public": None,
 922                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 923         if kwargs:
 924             # FORCE
 925             if "FORCE" in kwargs:
 926                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 927                     admin_query["force"] = True
 928                 del kwargs["FORCE"]
 929             # PUBLIC
 930             if "PUBLIC" in kwargs:
 931                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 932                     admin_query["public"] = True
 933                 else:
 934                     admin_query["public"] = False
 935                 del kwargs["PUBLIC"]
 936             # ADMIN
 937             if "ADMIN" in kwargs:
 938                 behave_as = kwargs.pop("ADMIN")
 939                 if behave_as.lower() != "false":
 940                     if not token_info["admin"]:
 941                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 942                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 943                         admin_query["project_id"] = ()
 944                     elif isinstance(behave_as, (list, tuple)):
 945                         admin_query["project_id"] = behave_as
 946                     else:   # isinstance(behave_as, str)
 947                         admin_query["project_id"] = (behave_as, )
 948             if "SET_PROJECT" in kwargs:
 949                 set_project = kwargs.pop("SET_PROJECT")
 950                 if not set_project:
 951                     admin_query["set_project"] = list(admin_query["project_id"])
 952                 else:
 953                     if isinstance(set_project, str):
 954                         set_project = (set_project, )
 955                     if admin_query["project_id"]:
 956                         for p in set_project:
 957                             if p not in admin_query["project_id"]:
 958                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 959                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 960                     admin_query["set_project"] = set_project
 961
 962             # PROJECT_READ
 963             # if "PROJECT_READ" in kwargs:
 964             #     admin_query["project"] = kwargs.pop("project")
 965             #     if admin_query["project"] == token_info["project_id"]:
 966         if method == "GET":
 967             if _id:
 968                 admin_query["method"] = "show"
 969             else:
 970                 admin_query["method"] = "list"
 971         elif method == "DELETE":
 972             admin_query["method"] = "delete"
 973         else:
 974             admin_query["method"] = "write"
 975         return admin_query
 976
 977     @cherrypy.expose
 978     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 979         token_info = None
 980         outdata = None
 981         _format = None
 982         method = "DONE"
 983         engine_topic = None
 984         rollback = []
 985         engine_session = None
 986         try:
 987             if not main_topic or not version or not topic:
 988                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 989                                    HTTPStatus.METHOD_NOT_ALLOWED)
 990             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 991                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 992                                    HTTPStatus.METHOD_NOT_ALLOWED)
 993             if version != 'v1':
 994                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 995
 996             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 997                 method = kwargs.pop("METHOD")
 998             else:
 999                 method = cherrypy.request.method
1000
1001             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
1002             query_string_operations = self._extract_query_string_operations(kwargs, method)
1003             if main_topic == "admin" and topic == "tokens":
1004                 return self.token(method, _id, kwargs)
1005             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
1006             if main_topic == "admin" and topic == "domains":
1007                 return self.domain()
1008             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
1009             indata = self._format_in(kwargs)
1010             engine_topic = topic
1011
1012             if item and topic != "pm_jobs":
1013                 engine_topic = item
1014
1015             if main_topic == "nsd":
1016                 engine_topic = "nsds"
1017             elif main_topic == "vnfpkgm":
1018                 engine_topic = "vnfds"
1019                 if topic == "vnfpkg_op_occs":
1020                     engine_topic = "vnfpkgops"
1021                 if topic == "vnf_packages" and item == "action":
1022                     engine_topic = "vnfpkgops"
1023             elif main_topic == "nslcm":
1024                 engine_topic = "nsrs"
1025                 if topic == "ns_lcm_op_occs":
1026                     engine_topic = "nslcmops"
1027                 if topic == "vnfrs" or topic == "vnf_instances":
1028                     engine_topic = "vnfrs"
1029             elif main_topic == "nst":
1030                 engine_topic = "nsts"
1031             elif main_topic == "nsilcm":
1032                 engine_topic = "nsis"
1033                 if topic == "nsi_lcm_op_occs":
1034                     engine_topic = "nsilcmops"
1035             elif main_topic == "pdu":
1036                 engine_topic = "pdus"
1037             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1038                 engine_topic = "vim_accounts"
1039
1040             if topic == "subscriptions":
1041                 engine_topic = main_topic + "_" + topic
1042
1043             if method == "GET":
1044                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1045                     if item in ("vnfd", "nsd", "nst"):
1046                         path = "$DESCRIPTOR"
1047                     elif args:
1048                         path = args
1049                     elif item == "artifacts":
1050                         path = ()
1051                     else:
1052                         path = None
1053                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1054                                                          cherrypy.request.headers.get("Accept"))
1055                     outdata = file
1056                 elif not _id:
1057                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs, api_req=True)
1058                 else:
1059                     if item == "reports":
1060                         # TODO check that project_id (_id in this context) has permissions
1061                         _id = args[0]
1062                     outdata = self.engine.get_item(engine_session, engine_topic, _id, True)
1063
1064             elif method == "POST":
1065                 cherrypy.response.status = HTTPStatus.CREATED.value
1066                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1067                     _id = cherrypy.request.headers.get("Transaction-Id")
1068                     if not _id:
1069                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1070                                                       cherrypy.request.headers)
1071                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1072                                                            cherrypy.request.headers)
1073                     if completed:
1074                         self._set_location_header(main_topic, version, topic, _id)
1075                     else:
1076                         cherrypy.response.headers["Transaction-Id"] = _id
1077                     outdata = {"id": _id}
1078                 elif topic == "ns_instances_content":
1079                     # creates NSR
1080                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1081                     # creates nslcmop
1082                     indata["lcmOperationType"] = "instantiate"
1083                     indata["nsInstanceId"] = _id
1084                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1085                     self._set_location_header(main_topic, version, topic, _id)
1086                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1087                 elif topic == "ns_instances" and item:
1088                     indata["lcmOperationType"] = item
1089                     indata["nsInstanceId"] = _id
1090                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1091                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1092                     outdata = {"id": _id}
1093                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1094                 elif topic == "netslice_instances_content":
1095                     # creates NetSlice_Instance_record (NSIR)
1096                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1097                     self._set_location_header(main_topic, version, topic, _id)
1098                     indata["lcmOperationType"] = "instantiate"
1099                     indata["netsliceInstanceId"] = _id
1100                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1101                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1102                 elif topic == "netslice_instances" and item:
1103                     indata["lcmOperationType"] = item
1104                     indata["netsliceInstanceId"] = _id
1105                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1106                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1107                     outdata = {"id": _id}
1108                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1109                 elif topic == "vnf_packages" and item == "action":
1110                     indata["lcmOperationType"] = item
1111                     indata["vnfPkgId"] = _id
1112                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1113                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1114                     outdata = {"id": _id}
1115                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1116                 elif topic == "subscriptions":
1117                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1118                     self._set_location_header(main_topic, version, topic, _id)
1119                     link = {}
1120                     link["self"] = cherrypy.response.headers["Location"]
1121                     outdata = {"id": _id, "filter": indata["filter"], "callbackUri": indata["CallbackUri"],
1122                                "_links": link}
1123                     cherrypy.response.status = HTTPStatus.CREATED.value
1124                 else:
1125                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1126                                                       cherrypy.request.headers)
1127                     self._set_location_header(main_topic, version, topic, _id)
1128                     outdata = {"id": _id}
1129                     if op_id:
1130                         outdata["op_id"] = op_id
1131                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1132                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1133
1134             elif method == "DELETE":
1135                 if not _id:
1136                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1137                     cherrypy.response.status = HTTPStatus.OK.value
1138                 else:  # len(args) > 1
1139                     # for NS NSI generate an operation
1140                     op_id = None
1141                     if topic == "ns_instances_content" and not engine_session["force"]:
1142                         nslcmop_desc = {
1143                             "lcmOperationType": "terminate",
1144                             "nsInstanceId": _id,
1145                             "autoremove": True
1146                         }
1147                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1148                         if op_id:
1149                             outdata = {"_id": op_id}
1150                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1151                         nsilcmop_desc = {
1152                             "lcmOperationType": "terminate",
1153                             "netsliceInstanceId": _id,
1154                             "autoremove": True
1155                         }
1156                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1157                         if op_id:
1158                             outdata = {"_id": op_id}
1159                     # if there is not any deletion in process, delete
1160                     if not op_id:
1161                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1162                         if op_id:
1163                             outdata = {"op_id": op_id}
1164                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1165
1166             elif method in ("PUT", "PATCH"):
1167                 op_id = None
1168                 if not indata and not kwargs and not engine_session.get("set_project"):
1169                     raise NbiException("Nothing to update. Provide payload and/or query string",
1170                                        HTTPStatus.BAD_REQUEST)
1171                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1172                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1173                                                            cherrypy.request.headers)
1174                     if not completed:
1175                         cherrypy.response.headers["Transaction-Id"] = id
1176                 else:
1177                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1178
1179                 if op_id:
1180                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1181                     outdata = {"op_id": op_id}
1182                 else:
1183                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1184                     outdata = None
1185             else:
1186                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1187
1188             # if Role information changes, it is needed to reload the information of roles
1189             if topic == "roles" and method != "GET":
1190                 self.authenticator.load_operation_to_allowed_roles()
1191
1192             if topic == "projects" and method == "DELETE" \
1193                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1194                 self.authenticator.remove_token_from_cache()
1195
1196             return self._format_out(outdata, token_info, _format)
1197         except Exception as e:
1198             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1199                               ValidationError, AuthconnException)):
1200                 http_code_value = cherrypy.response.status = e.http_code.value
1201                 http_code_name = e.http_code.name
1202                 cherrypy.log("Exception {}".format(e))
1203             else:
1204                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1205                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1206                 http_code_name = HTTPStatus.BAD_REQUEST.name
1207             if hasattr(outdata, "close"):  # is an open file
1208                 outdata.close()
1209             error_text = str(e)
1210             rollback.reverse()
1211             for rollback_item in rollback:
1212                 try:
1213                     if rollback_item.get("operation") == "set":
1214                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1215                                                rollback_item["content"], fail_on_empty=False)
1216                     elif rollback_item.get("operation") == "del_list":
1217                         self.engine.db.del_list(rollback_item["topic"], rollback_item["filter"],
1218                                                 fail_on_empty=False)
1219                     else:
1220                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1221                                                fail_on_empty=False)
1222                 except Exception as e2:
1223                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1224                     cherrypy.log(rollback_error_text)
1225                     error_text += ". " + rollback_error_text
1226             # if isinstance(e, MsgException):
1227             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1228             #         engine_topic[:-1], method, error_text)
1229             problem_details = {
1230                 "code": http_code_name,
1231                 "status": http_code_value,
1232                 "detail": error_text,
1233             }
1234             return self._format_out(problem_details, token_info)
1235             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1236         finally:
1237             if token_info:
1238                 self._format_login(token_info)
1239                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1240                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1241                         if outdata.get(logging_id):
1242                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1243
1244
1245 def _start_service():
1246     """
1247     Callback function called when cherrypy.engine starts
1248     Override configuration with env variables
1249     Set database, storage, message configuration
1250     Init database with admin/admin user password
1251     """
1252     global nbi_server
1253     global subscription_thread
1254     cherrypy.log.error("Starting osm_nbi")
1255     # update general cherrypy configuration
1256     update_dict = {}
1257
1258     engine_config = cherrypy.tree.apps['/osm'].config
1259     for k, v in environ.items():
1260         if not k.startswith("OSMNBI_"):
1261             continue
1262         k1, _, k2 = k[7:].lower().partition("_")
1263         if not k2:
1264             continue
1265         try:
1266             # update static configuration
1267             if k == 'OSMNBI_STATIC_DIR':
1268                 engine_config["/static"]['tools.staticdir.dir'] = v
1269                 engine_config["/static"]['tools.staticdir.on'] = True
1270             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1271                 update_dict['server.socket_port'] = int(v)
1272             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1273                 update_dict['server.socket_host'] = v
1274             elif k1 in ("server", "test", "auth", "log"):
1275                 update_dict[k1 + '.' + k2] = v
1276             elif k1 in ("message", "database", "storage", "authentication"):
1277                 # k2 = k2.replace('_', '.')
1278                 if k2 in ("port", "db_port"):
1279                     engine_config[k1][k2] = int(v)
1280                 else:
1281                     engine_config[k1][k2] = v
1282
1283         except ValueError as e:
1284             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1285         except Exception as e:
1286             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1287
1288     if update_dict:
1289         cherrypy.config.update(update_dict)
1290         engine_config["global"].update(update_dict)
1291
1292     # logging cherrypy
1293     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1294     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1295     logger_server = logging.getLogger("cherrypy.error")
1296     logger_access = logging.getLogger("cherrypy.access")
1297     logger_cherry = logging.getLogger("cherrypy")
1298     logger_nbi = logging.getLogger("nbi")
1299
1300     if "log.file" in engine_config["global"]:
1301         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1302                                                             maxBytes=100e6, backupCount=9, delay=0)
1303         file_handler.setFormatter(log_formatter_simple)
1304         logger_cherry.addHandler(file_handler)
1305         logger_nbi.addHandler(file_handler)
1306     # log always to standard output
1307     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1308                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1309                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1310                             }.items():
1311         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1312         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1313         str_handler = logging.StreamHandler()
1314         str_handler.setFormatter(log_formatter_cherry)
1315         logger.addHandler(str_handler)
1316
1317     if engine_config["global"].get("log.level"):
1318         logger_cherry.setLevel(engine_config["global"]["log.level"])
1319         logger_nbi.setLevel(engine_config["global"]["log.level"])
1320
1321     # logging other modules
1322     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1323         engine_config[k1]["logger_name"] = logname
1324         logger_module = logging.getLogger(logname)
1325         if "logfile" in engine_config[k1]:
1326             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1327                                                                 maxBytes=100e6, backupCount=9, delay=0)
1328             file_handler.setFormatter(log_formatter_simple)
1329             logger_module.addHandler(file_handler)
1330         if "loglevel" in engine_config[k1]:
1331             logger_module.setLevel(engine_config[k1]["loglevel"])
1332     # TODO add more entries, e.g.: storage
1333     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1334     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1335     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1336     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1337
1338     # start subscriptions thread:
1339     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1340     subscription_thread.start()
1341     # Do not capture except SubscriptionException
1342
1343     backend = engine_config["authentication"]["backend"]
1344     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1345                        .format(nbi_version, nbi_version_date, backend))
1346
1347
1348 def _stop_service():
1349     """
1350     Callback function called when cherrypy.engine stops
1351     TODO: Ending database connections.
1352     """
1353     global subscription_thread
1354     if subscription_thread:
1355         subscription_thread.terminate()
1356     subscription_thread = None
1357     cherrypy.tree.apps['/osm'].root.engine.stop()
1358     cherrypy.log.error("Stopping osm_nbi")
1359
1360
1361 def nbi(config_file):
1362     global nbi_server
1363     # conf = {
1364     #     '/': {
1365     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1366     #         'tools.sessions.on': True,
1367     #         'tools.response_headers.on': True,
1368     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1369     #     }
1370     # }
1371     # cherrypy.Server.ssl_module = 'builtin'
1372     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1373     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1374     # cherrypy.Server.thread_pool = 10
1375     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1376
1377     # cherrypy.config.update({'tools.auth_basic.on': True,
1378     #    'tools.auth_basic.realm': 'localhost',
1379     #    'tools.auth_basic.checkpassword': validate_password})
1380     nbi_server = Server()
1381     cherrypy.engine.subscribe('start', _start_service)
1382     cherrypy.engine.subscribe('stop', _stop_service)
1383     cherrypy.quickstart(nbi_server, '/osm', config_file)
1384
1385
1386 def usage():
1387     print("""Usage: {} [options]
1388         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1389         -h|--help: shows this help
1390         """.format(sys.argv[0]))
1391     # --log-socket-host HOST: send logs to this host")
1392     # --log-socket-port PORT: send logs using this port (default: 9022)")
1393
1394
1395 if __name__ == '__main__':
1396     try:
1397         # load parameters and configuration
1398         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1399         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1400         config_file = None
1401         for o, a in opts:
1402             if o in ("-h", "--help"):
1403                 usage()
1404                 sys.exit()
1405             elif o in ("-c", "--config"):
1406                 config_file = a
1407             # elif o == "--log-socket-port":
1408             #     log_socket_port = a
1409             # elif o == "--log-socket-host":
1410             #     log_socket_host = a
1411             # elif o == "--log-file":
1412             #     log_file = a
1413             else:
1414                 assert False, "Unhandled option"
1415         if config_file:
1416             if not path.isfile(config_file):
1417                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1418                 exit(1)
1419         else:
1420             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1421                 if path.isfile(config_file):
1422                     break
1423             else:
1424                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1425                 exit(1)
1426         nbi(config_file)
1427     except getopt.GetoptError as e:
1428         print(str(e), file=sys.stderr)
1429         # usage()
1430         exit(1)