osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118
 119         /nst/v1                                                 O       O
 120             /netslice_templates_content                         O       O
 121                 /<nstInfoId>                                    O       O       O       O
 122             /netslice_templates                                 O       O
 123                 /<nstInfoId>                                    O                       O       O
 124                     /nst_content                                O               O
 125                     /nst                                        O
 126                     /artifacts[/<artifactPath>]                 O
 127             /subscriptions                                      X       X
 128                 /<subscriptionId>                               X                       X
 129
 130         /nsilcm/v1
 131             /netslice_instances_content                         O       O
 132                 /<SliceInstanceId>                              O                       O
 133             /netslice_instances                                 O       O
 134                 /<SliceInstanceId>                              O                       O
 135                     instantiate                                         O
 136                     terminate                                           O
 137                     action                                              O
 138             /nsi_lcm_op_occs                                    O       O
 139                 /<nsiLcmOpOccId>                                O                       O       O
 140             /subscriptions                                      X       X
 141                 /<subscriptionId>                               X                       X
 142
 143 query string:
 144     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 145         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 146         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 147         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 148         attrName := string
 149     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 150     item of the array, that is, pass if any item of the array pass the filter.
 151     It allows both ne and neq for not equal
 152     TODO: 4.3.3 Attribute selectors
 153         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 154         (none)  … same as “exclude_default”
 155         all_fields      … all attributes.
 156         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 157         conditionally mandatory, and that are not provided in <list>.
 158         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 159         are not conditionally mandatory, and that are provided in <list>.
 160         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 161         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 162         the particular resource
 163         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 164         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 165         present specification for the particular resource, but that are not part of <list>
 166     Additionally it admits some administrator values:
 167         FORCE: To force operations skipping dependency checkings
 168         ADMIN: To act as an administrator or a different project
 169         PUBLIC: To get public descriptors or set a descriptor as public
 170         SET_PROJECT: To make a descriptor available for other project
 171
 172 Header field name       Reference       Example Descriptions
 173     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 174     This header field shall be present if the response is expected to have a non-empty message body.
 175     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 176     This header field shall be present if the request has a non-empty message body.
 177     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 178     Details are specified in clause 4.5.3.
 179     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 180 Header field name       Reference       Example Descriptions
 181     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 182     This header field shall be present if the response has a non-empty message body.
 183     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 184     new resource has been created.
 185     This header field shall be present if the response status code is 201 or 3xx.
 186     In the present document this header field is also used if the response status code is 202 and a new resource was
 187     created.
 188     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 189     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 190     token.
 191     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 192     certain resources.
 193     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 194     response, and the total length of the file.
 195     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 196 """
 197
 198 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 199 # ^ Contains possible administrative query string words:
 200 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 201 #           (not owned by my session project).
 202 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 203 #     FORCE=True(by default)|False: Force edition/deletion operations
 204 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 205
 206 valid_url_methods = {
 207     # contains allowed URL and methods, and the role_permission name
 208     "admin": {
 209         "v1": {
 210             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 211                        "ROLE_PERMISSION": "tokens:",
 212                        "<ID>": {"METHODS": ("GET", "DELETE"),
 213                                 "ROLE_PERMISSION": "tokens:id:"
 214                                 }
 215                        },
 216             "users": {"METHODS": ("GET", "POST"),
 217                       "ROLE_PERMISSION": "users:",
 218                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 219                                "ROLE_PERMISSION": "users:id:"
 220                                }
 221                       },
 222             "projects": {"METHODS": ("GET", "POST"),
 223                          "ROLE_PERMISSION": "projects:",
 224                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 225                                   "ROLE_PERMISSION": "projects:id:"}
 226                          },
 227             "roles": {"METHODS": ("GET", "POST"),
 228                       "ROLE_PERMISSION": "roles:",
 229                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 230                                "ROLE_PERMISSION": "roles:id:"
 231                                }
 232                       },
 233             "vims": {"METHODS": ("GET", "POST"),
 234                      "ROLE_PERMISSION": "vims:",
 235                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 236                               "ROLE_PERMISSION": "vims:id:"
 237                               }
 238                      },
 239             "vim_accounts": {"METHODS": ("GET", "POST"),
 240                              "ROLE_PERMISSION": "vim_accounts:",
 241                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 242                                       "ROLE_PERMISSION": "vim_accounts:id:"
 243                                       }
 244                              },
 245             "wim_accounts": {"METHODS": ("GET", "POST"),
 246                              "ROLE_PERMISSION": "wim_accounts:",
 247                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 248                                       "ROLE_PERMISSION": "wim_accounts:id:"
 249                                       }
 250                              },
 251             "sdns": {"METHODS": ("GET", "POST"),
 252                      "ROLE_PERMISSION": "sdn_controllers:",
 253                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 254                               "ROLE_PERMISSION": "sdn_controllers:id:"
 255                               }
 256                      },
 257             "k8sclusters": {"METHODS": ("GET", "POST"),
 258                             "ROLE_PERMISSION": "k8sclusters:",
 259                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 260                                      "ROLE_PERMISSION": "k8sclusters:id:"
 261                                      }
 262                             },
 263             "k8srepos": {"METHODS": ("GET", "POST"),
 264                          "ROLE_PERMISSION": "k8srepos:",
 265                          "<ID>": {"METHODS": ("GET", "DELETE"),
 266                                   "ROLE_PERMISSION": "k8srepos:id:"
 267                                   }
 268                          },
 269
 270         }
 271     },
 272     "pdu": {
 273         "v1": {
 274             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 275                                 "ROLE_PERMISSION": "pduds:",
 276                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 277                                          "ROLE_PERMISSION": "pduds:id:"
 278                                          }
 279                                 },
 280         }
 281     },
 282     "nsd": {
 283         "v1": {
 284             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 285                                        "ROLE_PERMISSION": "nsds:",
 286                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 287                                                 "ROLE_PERMISSION": "nsds:id:"
 288                                                 }
 289                                        },
 290             "ns_descriptors": {"METHODS": ("GET", "POST"),
 291                                "ROLE_PERMISSION": "nsds:",
 292                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 293                                         "ROLE_PERMISSION": "nsds:id:",
 294                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 295                                                         "ROLE_PERMISSION": "nsds:id:content:",
 296                                                         },
 297                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 298                                                 "ROLE_PERMISSION": "nsds:id:content:"
 299                                                 },
 300                                         "artifacts": {"*": {"METHODS": ("GET",),
 301                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 302                                                             }
 303                                                       }
 304                                         }
 305                                },
 306             "pnf_descriptors": {"TODO": ("GET", "POST"),
 307                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 308                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 309                                          }
 310                                 },
 311             "subscriptions": {"TODO": ("GET", "POST"),
 312                               "<ID>": {"TODO": ("GET", "DELETE")}
 313                               },
 314         }
 315     },
 316     "vnfpkgm": {
 317         "v1": {
 318             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 319                                      "ROLE_PERMISSION": "vnfds:",
 320                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 321                                               "ROLE_PERMISSION": "vnfds:id:"}
 322                                      },
 323             "vnf_packages": {"METHODS": ("GET", "POST"),
 324                              "ROLE_PERMISSION": "vnfds:",
 325                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 326                                       "ROLE_PERMISSION": "vnfds:id:",
 327                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 328                                                           "ROLE_PERMISSION": "vnfds:id:",
 329                                                           "upload_from_uri": {"METHODS": (),
 330                                                                               "TODO": ("POST", ),
 331                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 332                                                                               }
 333                                                           },
 334                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 335                                                "ROLE_PERMISSION": "vnfds:id:content:"
 336                                                },
 337                                       "artifacts": {"*": {"METHODS": ("GET", ),
 338                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 339                                                           }
 340                                                     },
 341                                       "action": {"METHODS": ("POST", ),
 342                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 343                                                  },
 344                                       }
 345                              },
 346             "subscriptions": {"TODO": ("GET", "POST"),
 347                               "<ID>": {"TODO": ("GET", "DELETE")}
 348                               },
 349             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 350                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 351                                "<ID>": {"METHODS": ("GET", ),
 352                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 353                                         }
 354                                },
 355         }
 356     },
 357     "nslcm": {
 358         "v1": {
 359             "ns_instances_content": {"METHODS": ("GET", "POST"),
 360                                      "ROLE_PERMISSION": "ns_instances:",
 361                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 362                                               "ROLE_PERMISSION": "ns_instances:id:"
 363                                               }
 364                                      },
 365             "ns_instances": {"METHODS": ("GET", "POST"),
 366                              "ROLE_PERMISSION": "ns_instances:",
 367                              "<ID>": {"METHODS": ("GET", "DELETE"),
 368                                       "ROLE_PERMISSION": "ns_instances:id:",
 369                                       "scale": {"METHODS": ("POST",),
 370                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 371                                                 },
 372                                       "terminate": {"METHODS": ("POST",),
 373                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 374                                                     },
 375                                       "instantiate": {"METHODS": ("POST",),
 376                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 377                                                       },
 378                                       "action": {"METHODS": ("POST",),
 379                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 380                                                  },
 381                                       }
 382                              },
 383             "ns_lcm_op_occs": {"METHODS": ("GET",),
 384                                "ROLE_PERMISSION": "ns_instances:opps:",
 385                                "<ID>": {"METHODS": ("GET",),
 386                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 387                                         },
 388                                },
 389             "vnfrs": {"METHODS": ("GET",),
 390                       "ROLE_PERMISSION": "vnf_instances:",
 391                       "<ID>": {"METHODS": ("GET",),
 392                                "ROLE_PERMISSION": "vnf_instances:id:"
 393                                }
 394                       },
 395             "vnf_instances": {"METHODS": ("GET",),
 396                               "ROLE_PERMISSION": "vnf_instances:",
 397                               "<ID>": {"METHODS": ("GET",),
 398                                        "ROLE_PERMISSION": "vnf_instances:id:"
 399                                        }
 400                               },
 401         }
 402     },
 403     "nst": {
 404         "v1": {
 405             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 406                                            "ROLE_PERMISSION": "slice_templates:",
 407                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 408                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 409                                            },
 410             "netslice_templates": {"METHODS": ("GET", "POST"),
 411                                    "ROLE_PERMISSION": "slice_templates:",
 412                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 413                                             "TODO": ("PATCH",),
 414                                             "ROLE_PERMISSION": "slice_templates:id:",
 415                                             "nst_content": {"METHODS": ("GET", "PUT"),
 416                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 417                                                             },
 418                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 419                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 420                                                     },
 421                                             "artifacts": {"*": {"METHODS": ("GET",),
 422                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 423                                                                 }
 424                                                           }
 425                                             }
 426                                    },
 427             "subscriptions": {"TODO": ("GET", "POST"),
 428                               "<ID>": {"TODO": ("GET", "DELETE")}
 429                               },
 430         }
 431     },
 432     "nsilcm": {
 433         "v1": {
 434             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 435                                            "ROLE_PERMISSION": "slice_instances:",
 436                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 437                                                     "ROLE_PERMISSION": "slice_instances:id:"
 438                                                     }
 439                                            },
 440             "netslice_instances": {"METHODS": ("GET", "POST"),
 441                                    "ROLE_PERMISSION": "slice_instances:",
 442                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 443                                             "ROLE_PERMISSION": "slice_instances:id:",
 444                                             "terminate": {"METHODS": ("POST",),
 445                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 446                                                           },
 447                                             "instantiate": {"METHODS": ("POST",),
 448                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 449                                                             },
 450                                             "action": {"METHODS": ("POST",),
 451                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 452                                                        },
 453                                             }
 454                                    },
 455             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 456                                 "ROLE_PERMISSION": "slice_instances:opps:",
 457                                 "<ID>": {"METHODS": ("GET",),
 458                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 459                                          },
 460                                 },
 461         }
 462     },
 463     "nspm": {
 464         "v1": {
 465             "pm_jobs": {
 466                 "<ID>": {
 467                     "reports": {
 468                         "<ID>": {"METHODS": ("GET",),
 469                                  "ROLE_PERMISSION": "reports:id:",
 470                                  }
 471                     }
 472                 },
 473             },
 474         },
 475     },
 476 }
 477
 478
 479 class NbiException(Exception):
 480
 481     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 482         Exception.__init__(self, message)
 483         self.http_code = http_code
 484
 485
 486 class Server(object):
 487     instance = 0
 488     # to decode bytes to str
 489     reader = getreader("utf-8")
 490
 491     def __init__(self):
 492         self.instance += 1
 493         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 494         self.engine = Engine(self.authenticator)
 495
 496     def _format_in(self, kwargs):
 497         try:
 498             indata = None
 499             if cherrypy.request.body.length:
 500                 error_text = "Invalid input format "
 501
 502                 if "Content-Type" in cherrypy.request.headers:
 503                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 504                         error_text = "Invalid json format "
 505                         indata = json.load(self.reader(cherrypy.request.body))
 506                         cherrypy.request.headers.pop("Content-File-MD5", None)
 507                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 508                         error_text = "Invalid yaml format "
 509                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 510                         cherrypy.request.headers.pop("Content-File-MD5", None)
 511                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 512                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 513                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 514                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 515                         indata = cherrypy.request.body  # .read()
 516                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 517                         if "descriptor_file" in kwargs:
 518                             filecontent = kwargs.pop("descriptor_file")
 519                             if not filecontent.file:
 520                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 521                             indata = filecontent.file  # .read()
 522                             if filecontent.content_type.value:
 523                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 524                     else:
 525                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 526                         #                          "Only 'Content-Type' of type 'application/json' or
 527                         # 'application/yaml' for input format are available")
 528                         error_text = "Invalid yaml format "
 529                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 530                         cherrypy.request.headers.pop("Content-File-MD5", None)
 531                 else:
 532                     error_text = "Invalid yaml format "
 533                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 534                     cherrypy.request.headers.pop("Content-File-MD5", None)
 535             if not indata:
 536                 indata = {}
 537
 538             format_yaml = False
 539             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 540                 format_yaml = True
 541
 542             for k, v in kwargs.items():
 543                 if isinstance(v, str):
 544                     if v == "":
 545                         kwargs[k] = None
 546                     elif format_yaml:
 547                         try:
 548                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 549                         except Exception:
 550                             pass
 551                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 552                         try:
 553                             kwargs[k] = int(v)
 554                         except Exception:
 555                             try:
 556                                 kwargs[k] = float(v)
 557                             except Exception:
 558                                 pass
 559                     elif v.find(",") > 0:
 560                         kwargs[k] = v.split(",")
 561                 elif isinstance(v, (list, tuple)):
 562                     for index in range(0, len(v)):
 563                         if v[index] == "":
 564                             v[index] = None
 565                         elif format_yaml:
 566                             try:
 567                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 568                             except Exception:
 569                                 pass
 570
 571             return indata
 572         except (ValueError, yaml.YAMLError) as exc:
 573             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 574         except KeyError as exc:
 575             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 576         except Exception as exc:
 577             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 578
 579     @staticmethod
 580     def _format_out(data, token_info=None, _format=None):
 581         """
 582         return string of dictionary data according to requested json, yaml, xml. By default json
 583         :param data: response to be sent. Can be a dict, text or file
 584         :param token_info: Contains among other username and project
 585         :param _format: The format to be set as Content-Type if data is a file
 586         :return: None
 587         """
 588         accept = cherrypy.request.headers.get("Accept")
 589         if data is None:
 590             if accept and "text/html" in accept:
 591                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 592             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 593             return
 594         elif hasattr(data, "read"):  # file object
 595             if _format:
 596                 cherrypy.response.headers["Content-Type"] = _format
 597             elif "b" in data.mode:  # binariy asssumig zip
 598                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 599             else:
 600                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 601             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 602             return data
 603         if accept:
 604             if "application/json" in accept:
 605                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 606                 a = json.dumps(data, indent=4) + "\n"
 607                 return a.encode("utf8")
 608             elif "text/html" in accept:
 609                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 610
 611             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 612                 pass
 613             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 614             elif cherrypy.response.status >= 400:
 615                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 616                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 617                                          "for output format are available")
 618         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 619         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 620                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 621
 622     @cherrypy.expose
 623     def index(self, *args, **kwargs):
 624         token_info = None
 625         try:
 626             if cherrypy.request.method == "GET":
 627                 token_info = self.authenticator.authorize()
 628                 outdata = token_info   # Home page
 629             else:
 630                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 631                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 632
 633             return self._format_out(outdata, token_info)
 634
 635         except (EngineException, AuthException) as e:
 636             # cherrypy.log("index Exception {}".format(e))
 637             cherrypy.response.status = e.http_code.value
 638             return self._format_out("Welcome to OSM!", token_info)
 639
 640     @cherrypy.expose
 641     def version(self, *args, **kwargs):
 642         # TODO consider to remove and provide version using the static version file
 643         try:
 644             if cherrypy.request.method != "GET":
 645                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 646             elif args or kwargs:
 647                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 648             # TODO include version of other modules, pick up from some kafka admin message
 649             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 650             return self._format_out(osm_nbi_version)
 651         except NbiException as e:
 652             cherrypy.response.status = e.http_code.value
 653             problem_details = {
 654                 "code": e.http_code.name,
 655                 "status": e.http_code.value,
 656                 "detail": str(e),
 657             }
 658             return self._format_out(problem_details, None)
 659
 660     @staticmethod
 661     def _format_login(token_info):
 662         """
 663         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 664         log this information
 665         :param token_info: Dictionary with token content
 666         :return: None
 667         """
 668         cherrypy.request.login = token_info.get("username", "-")
 669         if token_info.get("project_name"):
 670             cherrypy.request.login += "/" + token_info["project_name"]
 671         if token_info.get("id"):
 672             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 673
 674     @cherrypy.expose
 675     def token(self, method, token_id=None, kwargs=None):
 676         token_info = None
 677         # self.engine.load_dbase(cherrypy.request.app.config)
 678         indata = self._format_in(kwargs)
 679         if not isinstance(indata, dict):
 680             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 681
 682         if method == "GET":
 683             token_info = self.authenticator.authorize()
 684             # for logging
 685             self._format_login(token_info)
 686             if token_id:
 687                 outdata = self.authenticator.get_token(token_info, token_id)
 688             else:
 689                 outdata = self.authenticator.get_token_list(token_info)
 690         elif method == "POST":
 691             try:
 692                 token_info = self.authenticator.authorize()
 693             except Exception:
 694                 token_info = None
 695             if kwargs:
 696                 indata.update(kwargs)
 697             # This is needed to log the user when authentication fails
 698             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 699             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 700             cherrypy.session['Authorization'] = outdata["_id"]
 701             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 702             # for logging
 703             self._format_login(token_info)
 704
 705             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 706             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 707         elif method == "DELETE":
 708             if not token_id and "id" in kwargs:
 709                 token_id = kwargs["id"]
 710             elif not token_id:
 711                 token_info = self.authenticator.authorize()
 712                 # for logging
 713                 self._format_login(token_info)
 714                 token_id = token_info["_id"]
 715             outdata = self.authenticator.del_token(token_id)
 716             token_info = None
 717             cherrypy.session['Authorization'] = "logout"
 718             # cherrypy.response.cookie["Authorization"] = token_id
 719             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 720         else:
 721             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 722         return self._format_out(outdata, token_info)
 723
 724     @cherrypy.expose
 725     def test(self, *args, **kwargs):
 726         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 727                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 728             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 729             return "test URL is disabled"
 730         thread_info = None
 731         if args and args[0] == "help":
 732             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 733                    "sleep/<time>\nmessage/topic\n</pre></html>"
 734
 735         elif args and args[0] == "init":
 736             try:
 737                 # self.engine.load_dbase(cherrypy.request.app.config)
 738                 self.engine.create_admin()
 739                 return "Done. User 'admin', password 'admin' created"
 740             except Exception:
 741                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 742                 return self._format_out("Database already initialized")
 743         elif args and args[0] == "file":
 744             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 745                                                   "text/plain", "attachment")
 746         elif args and args[0] == "file2":
 747             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 748             f = open(f_path, "r")
 749             cherrypy.response.headers["Content-type"] = "text/plain"
 750             return f
 751
 752         elif len(args) == 2 and args[0] == "db-clear":
 753             deleted_info = self.engine.db.del_list(args[1], kwargs)
 754             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 755         elif len(args) and args[0] == "fs-clear":
 756             if len(args) >= 2:
 757                 folders = (args[1],)
 758             else:
 759                 folders = self.engine.fs.dir_ls(".")
 760             for folder in folders:
 761                 self.engine.fs.file_delete(folder)
 762             return ",".join(folders) + " folders deleted\n"
 763         elif args and args[0] == "login":
 764             if not cherrypy.request.headers.get("Authorization"):
 765                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 766                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 767         elif args and args[0] == "login2":
 768             if not cherrypy.request.headers.get("Authorization"):
 769                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 770                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 771         elif args and args[0] == "sleep":
 772             sleep_time = 5
 773             try:
 774                 sleep_time = int(args[1])
 775             except Exception:
 776                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 777                 return self._format_out("Database already initialized")
 778             thread_info = cherrypy.thread_data
 779             print(thread_info)
 780             time.sleep(sleep_time)
 781             # thread_info
 782         elif len(args) >= 2 and args[0] == "message":
 783             main_topic = args[1]
 784             return_text = "<html><pre>{} ->\n".format(main_topic)
 785             try:
 786                 if cherrypy.request.method == 'POST':
 787                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 788                     for k, v in to_send.items():
 789                         self.engine.msg.write(main_topic, k, v)
 790                         return_text += "  {}: {}\n".format(k, v)
 791                 elif cherrypy.request.method == 'GET':
 792                     for k, v in kwargs.items():
 793                         self.engine.msg.write(main_topic, k, yaml.load(v), Loader=yaml.SafeLoader)
 794                         return_text += "  {}: {}\n".format(k, yaml.load(v), Loader=yaml.SafeLoader)
 795             except Exception as e:
 796                 return_text += "Error: " + str(e)
 797             return_text += "</pre></html>\n"
 798             return return_text
 799
 800         return_text = (
 801             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 802             "  kwargs: {}\n".format(kwargs) +
 803             "  headers: {}\n".format(cherrypy.request.headers) +
 804             "  path_info: {}\n".format(cherrypy.request.path_info) +
 805             "  query_string: {}\n".format(cherrypy.request.query_string) +
 806             "  session: {}\n".format(cherrypy.session) +
 807             "  cookie: {}\n".format(cherrypy.request.cookie) +
 808             "  method: {}\n".format(cherrypy.request.method) +
 809             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 810             "  body:\n")
 811         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 812         if cherrypy.request.body.length:
 813             return_text += "    content: {}\n".format(
 814                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 815         if thread_info:
 816             return_text += "thread: {}\n".format(thread_info)
 817         return_text += "</pre></html>"
 818         return return_text
 819
 820     @staticmethod
 821     def _check_valid_url_method(method, *args):
 822         if len(args) < 3:
 823             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 824
 825         reference = valid_url_methods
 826         for arg in args:
 827             if arg is None:
 828                 break
 829             if not isinstance(reference, dict):
 830                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 831                                    HTTPStatus.METHOD_NOT_ALLOWED)
 832
 833             if arg in reference:
 834                 reference = reference[arg]
 835             elif "<ID>" in reference:
 836                 reference = reference["<ID>"]
 837             elif "*" in reference:
 838                 reference = reference["*"]
 839                 break
 840             else:
 841                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 842         if "TODO" in reference and method in reference["TODO"]:
 843             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 844         elif "METHODS" in reference and method not in reference["METHODS"]:
 845             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 846         return reference["ROLE_PERMISSION"] + method.lower()
 847
 848     @staticmethod
 849     def _set_location_header(main_topic, version, topic, id):
 850         """
 851         Insert response header Location with the URL of created item base on URL params
 852         :param main_topic:
 853         :param version:
 854         :param topic:
 855         :param id:
 856         :return: None
 857         """
 858         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 859         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 860         return
 861
 862     @staticmethod
 863     def _extract_query_string_operations(kwargs, method):
 864         """
 865
 866         :param kwargs:
 867         :return:
 868         """
 869         query_string_operations = []
 870         if kwargs:
 871             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 872                 if qs in kwargs and kwargs[qs].lower() != "false":
 873                     query_string_operations.append(qs.lower() + ":" + method.lower())
 874         return query_string_operations
 875
 876     @staticmethod
 877     def _manage_admin_query(token_info, kwargs, method, _id):
 878         """
 879         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 880         Check that users has rights to use them and returs the admin_query
 881         :param token_info: token_info rights obtained by token
 882         :param kwargs: query string input.
 883         :param method: http method: GET, POSST, PUT, ...
 884         :param _id:
 885         :return: admin_query dictionary with keys:
 886             public: True, False or None
 887             force: True or False
 888             project_id: tuple with projects used for accessing an element
 889             set_project: tuple with projects that a created element will belong to
 890             method: show, list, delete, write
 891         """
 892         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 893                        "admin": token_info["admin"], "public": None,
 894                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 895         if kwargs:
 896             # FORCE
 897             if "FORCE" in kwargs:
 898                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 899                     admin_query["force"] = True
 900                 del kwargs["FORCE"]
 901             # PUBLIC
 902             if "PUBLIC" in kwargs:
 903                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 904                     admin_query["public"] = True
 905                 else:
 906                     admin_query["public"] = False
 907                 del kwargs["PUBLIC"]
 908             # ADMIN
 909             if "ADMIN" in kwargs:
 910                 behave_as = kwargs.pop("ADMIN")
 911                 if behave_as.lower() != "false":
 912                     if not token_info["admin"]:
 913                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 914                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 915                         admin_query["project_id"] = ()
 916                     elif isinstance(behave_as, (list, tuple)):
 917                         admin_query["project_id"] = behave_as
 918                     else:   # isinstance(behave_as, str)
 919                         admin_query["project_id"] = (behave_as, )
 920             if "SET_PROJECT" in kwargs:
 921                 set_project = kwargs.pop("SET_PROJECT")
 922                 if not set_project:
 923                     admin_query["set_project"] = list(admin_query["project_id"])
 924                 else:
 925                     if isinstance(set_project, str):
 926                         set_project = (set_project, )
 927                     if admin_query["project_id"]:
 928                         for p in set_project:
 929                             if p not in admin_query["project_id"]:
 930                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 931                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 932                     admin_query["set_project"] = set_project
 933
 934             # PROJECT_READ
 935             # if "PROJECT_READ" in kwargs:
 936             #     admin_query["project"] = kwargs.pop("project")
 937             #     if admin_query["project"] == token_info["project_id"]:
 938         if method == "GET":
 939             if _id:
 940                 admin_query["method"] = "show"
 941             else:
 942                 admin_query["method"] = "list"
 943         elif method == "DELETE":
 944             admin_query["method"] = "delete"
 945         else:
 946             admin_query["method"] = "write"
 947         return admin_query
 948
 949     @cherrypy.expose
 950     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 951         token_info = None
 952         outdata = None
 953         _format = None
 954         method = "DONE"
 955         engine_topic = None
 956         rollback = []
 957         engine_session = None
 958         try:
 959             if not main_topic or not version or not topic:
 960                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 961                                    HTTPStatus.METHOD_NOT_ALLOWED)
 962             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 963                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 964                                    HTTPStatus.METHOD_NOT_ALLOWED)
 965             if version != 'v1':
 966                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 967
 968             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 969                 method = kwargs.pop("METHOD")
 970             else:
 971                 method = cherrypy.request.method
 972
 973             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 974             query_string_operations = self._extract_query_string_operations(kwargs, method)
 975             if main_topic == "admin" and topic == "tokens":
 976                 return self.token(method, _id, kwargs)
 977             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
 978             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 979             indata = self._format_in(kwargs)
 980             engine_topic = topic
 981             if topic == "subscriptions":
 982                 engine_topic = main_topic + "_" + topic
 983             if item and topic != "pm_jobs":
 984                 engine_topic = item
 985
 986             if main_topic == "nsd":
 987                 engine_topic = "nsds"
 988             elif main_topic == "vnfpkgm":
 989                 engine_topic = "vnfds"
 990                 if topic == "vnfpkg_op_occs":
 991                     engine_topic = "vnfpkgops"
 992                 if topic == "vnf_packages" and item == "action":
 993                     engine_topic = "vnfpkgops"
 994             elif main_topic == "nslcm":
 995                 engine_topic = "nsrs"
 996                 if topic == "ns_lcm_op_occs":
 997                     engine_topic = "nslcmops"
 998                 if topic == "vnfrs" or topic == "vnf_instances":
 999                     engine_topic = "vnfrs"
1000             elif main_topic == "nst":
1001                 engine_topic = "nsts"
1002             elif main_topic == "nsilcm":
1003                 engine_topic = "nsis"
1004                 if topic == "nsi_lcm_op_occs":
1005                     engine_topic = "nsilcmops"
1006             elif main_topic == "pdu":
1007                 engine_topic = "pdus"
1008             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1009                 engine_topic = "vim_accounts"
1010
1011             if method == "GET":
1012                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1013                     if item in ("vnfd", "nsd", "nst"):
1014                         path = "$DESCRIPTOR"
1015                     elif args:
1016                         path = args
1017                     elif item == "artifacts":
1018                         path = ()
1019                     else:
1020                         path = None
1021                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1022                                                          cherrypy.request.headers.get("Accept"))
1023                     outdata = file
1024                 elif not _id:
1025                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1026                 else:
1027                     if item == "reports":
1028                         # TODO check that project_id (_id in this context) has permissions
1029                         _id = args[0]
1030                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1031
1032             elif method == "POST":
1033                 cherrypy.response.status = HTTPStatus.CREATED.value
1034                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1035                     _id = cherrypy.request.headers.get("Transaction-Id")
1036                     if not _id:
1037                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1038                                                       cherrypy.request.headers)
1039                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1040                                                            cherrypy.request.headers)
1041                     if completed:
1042                         self._set_location_header(main_topic, version, topic, _id)
1043                     else:
1044                         cherrypy.response.headers["Transaction-Id"] = _id
1045                     outdata = {"id": _id}
1046                 elif topic == "ns_instances_content":
1047                     # creates NSR
1048                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1049                     # creates nslcmop
1050                     indata["lcmOperationType"] = "instantiate"
1051                     indata["nsInstanceId"] = _id
1052                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1053                     self._set_location_header(main_topic, version, topic, _id)
1054                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1055                 elif topic == "ns_instances" and item:
1056                     indata["lcmOperationType"] = item
1057                     indata["nsInstanceId"] = _id
1058                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1059                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1060                     outdata = {"id": _id}
1061                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1062                 elif topic == "netslice_instances_content":
1063                     # creates NetSlice_Instance_record (NSIR)
1064                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1065                     self._set_location_header(main_topic, version, topic, _id)
1066                     indata["lcmOperationType"] = "instantiate"
1067                     indata["netsliceInstanceId"] = _id
1068                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1069                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1070                 elif topic == "netslice_instances" and item:
1071                     indata["lcmOperationType"] = item
1072                     indata["netsliceInstanceId"] = _id
1073                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1074                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1075                     outdata = {"id": _id}
1076                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1077                 elif topic == "vnf_packages" and item == "action":
1078                     indata["lcmOperationType"] = item
1079                     indata["vnfPkgId"] = _id
1080                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1081                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1082                     outdata = {"id": _id}
1083                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1084                 else:
1085                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1086                                                       cherrypy.request.headers)
1087                     self._set_location_header(main_topic, version, topic, _id)
1088                     outdata = {"id": _id}
1089                     if op_id:
1090                         outdata["op_id"] = op_id
1091                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1092                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1093
1094             elif method == "DELETE":
1095                 if not _id:
1096                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1097                     cherrypy.response.status = HTTPStatus.OK.value
1098                 else:  # len(args) > 1
1099                     delete_in_process = False
1100                     if topic == "ns_instances_content" and not engine_session["force"]:
1101                         nslcmop_desc = {
1102                             "lcmOperationType": "terminate",
1103                             "nsInstanceId": _id,
1104                             "autoremove": True
1105                         }
1106                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, None)
1107                         if opp_id:
1108                             delete_in_process = True
1109                             outdata = {"_id": opp_id}
1110                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1111                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1112                         nsilcmop_desc = {
1113                             "lcmOperationType": "terminate",
1114                             "netsliceInstanceId": _id,
1115                             "autoremove": True
1116                         }
1117                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1118                         if opp_id:
1119                             delete_in_process = True
1120                             outdata = {"_id": opp_id}
1121                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1122                     if not delete_in_process:
1123                         self.engine.del_item(engine_session, engine_topic, _id)
1124                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1125                 if engine_topic in ("vim_accounts", "wim_accounts", "sdns", "k8sclusters", "k8srepos"):
1126                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1127
1128             elif method in ("PUT", "PATCH"):
1129                 op_id = None
1130                 if not indata and not kwargs and not engine_session.get("set_project"):
1131                     raise NbiException("Nothing to update. Provide payload and/or query string",
1132                                        HTTPStatus.BAD_REQUEST)
1133                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1134                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1135                                                            cherrypy.request.headers)
1136                     if not completed:
1137                         cherrypy.response.headers["Transaction-Id"] = id
1138                 else:
1139                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1140
1141                 if op_id:
1142                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1143                     outdata = {"op_id": op_id}
1144                 else:
1145                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1146                     outdata = None
1147             else:
1148                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1149
1150             # if Role information changes, it is needed to reload the information of roles
1151             if topic == "roles" and method != "GET":
1152                 self.authenticator.load_operation_to_allowed_roles()
1153
1154             if topic == "projects" and method == "DELETE" \
1155                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1156                 self.authenticator.remove_token_from_cache()
1157
1158             return self._format_out(outdata, token_info, _format)
1159         except Exception as e:
1160             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1161                               ValidationError, AuthconnException)):
1162                 http_code_value = cherrypy.response.status = e.http_code.value
1163                 http_code_name = e.http_code.name
1164                 cherrypy.log("Exception {}".format(e))
1165             else:
1166                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1167                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1168                 http_code_name = HTTPStatus.BAD_REQUEST.name
1169             if hasattr(outdata, "close"):  # is an open file
1170                 outdata.close()
1171             error_text = str(e)
1172             rollback.reverse()
1173             for rollback_item in rollback:
1174                 try:
1175                     if rollback_item.get("operation") == "set":
1176                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1177                                                rollback_item["content"], fail_on_empty=False)
1178                     else:
1179                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1180                                                fail_on_empty=False)
1181                 except Exception as e2:
1182                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1183                     cherrypy.log(rollback_error_text)
1184                     error_text += ". " + rollback_error_text
1185             # if isinstance(e, MsgException):
1186             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1187             #         engine_topic[:-1], method, error_text)
1188             problem_details = {
1189                 "code": http_code_name,
1190                 "status": http_code_value,
1191                 "detail": error_text,
1192             }
1193             return self._format_out(problem_details, token_info)
1194             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1195         finally:
1196             if token_info:
1197                 self._format_login(token_info)
1198                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1199                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1200                         if outdata.get(logging_id):
1201                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1202
1203
1204 def _start_service():
1205     """
1206     Callback function called when cherrypy.engine starts
1207     Override configuration with env variables
1208     Set database, storage, message configuration
1209     Init database with admin/admin user password
1210     """
1211     global nbi_server
1212     global subscription_thread
1213     cherrypy.log.error("Starting osm_nbi")
1214     # update general cherrypy configuration
1215     update_dict = {}
1216
1217     engine_config = cherrypy.tree.apps['/osm'].config
1218     for k, v in environ.items():
1219         if not k.startswith("OSMNBI_"):
1220             continue
1221         k1, _, k2 = k[7:].lower().partition("_")
1222         if not k2:
1223             continue
1224         try:
1225             # update static configuration
1226             if k == 'OSMNBI_STATIC_DIR':
1227                 engine_config["/static"]['tools.staticdir.dir'] = v
1228                 engine_config["/static"]['tools.staticdir.on'] = True
1229             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1230                 update_dict['server.socket_port'] = int(v)
1231             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1232                 update_dict['server.socket_host'] = v
1233             elif k1 in ("server", "test", "auth", "log"):
1234                 update_dict[k1 + '.' + k2] = v
1235             elif k1 in ("message", "database", "storage", "authentication"):
1236                 # k2 = k2.replace('_', '.')
1237                 if k2 in ("port", "db_port"):
1238                     engine_config[k1][k2] = int(v)
1239                 else:
1240                     engine_config[k1][k2] = v
1241
1242         except ValueError as e:
1243             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1244         except Exception as e:
1245             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1246
1247     if update_dict:
1248         cherrypy.config.update(update_dict)
1249         engine_config["global"].update(update_dict)
1250
1251     # logging cherrypy
1252     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1253     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1254     logger_server = logging.getLogger("cherrypy.error")
1255     logger_access = logging.getLogger("cherrypy.access")
1256     logger_cherry = logging.getLogger("cherrypy")
1257     logger_nbi = logging.getLogger("nbi")
1258
1259     if "log.file" in engine_config["global"]:
1260         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1261                                                             maxBytes=100e6, backupCount=9, delay=0)
1262         file_handler.setFormatter(log_formatter_simple)
1263         logger_cherry.addHandler(file_handler)
1264         logger_nbi.addHandler(file_handler)
1265     # log always to standard output
1266     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1267                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1268                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1269                             }.items():
1270         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1271         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1272         str_handler = logging.StreamHandler()
1273         str_handler.setFormatter(log_formatter_cherry)
1274         logger.addHandler(str_handler)
1275
1276     if engine_config["global"].get("log.level"):
1277         logger_cherry.setLevel(engine_config["global"]["log.level"])
1278         logger_nbi.setLevel(engine_config["global"]["log.level"])
1279
1280     # logging other modules
1281     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1282         engine_config[k1]["logger_name"] = logname
1283         logger_module = logging.getLogger(logname)
1284         if "logfile" in engine_config[k1]:
1285             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1286                                                                 maxBytes=100e6, backupCount=9, delay=0)
1287             file_handler.setFormatter(log_formatter_simple)
1288             logger_module.addHandler(file_handler)
1289         if "loglevel" in engine_config[k1]:
1290             logger_module.setLevel(engine_config[k1]["loglevel"])
1291     # TODO add more entries, e.g.: storage
1292     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1293     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1294     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1295     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1296
1297     # start subscriptions thread:
1298     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1299     subscription_thread.start()
1300     # Do not capture except SubscriptionException
1301
1302     backend = engine_config["authentication"]["backend"]
1303     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1304                        .format(nbi_version, nbi_version_date, backend))
1305
1306
1307 def _stop_service():
1308     """
1309     Callback function called when cherrypy.engine stops
1310     TODO: Ending database connections.
1311     """
1312     global subscription_thread
1313     if subscription_thread:
1314         subscription_thread.terminate()
1315     subscription_thread = None
1316     cherrypy.tree.apps['/osm'].root.engine.stop()
1317     cherrypy.log.error("Stopping osm_nbi")
1318
1319
1320 def nbi(config_file):
1321     global nbi_server
1322     # conf = {
1323     #     '/': {
1324     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1325     #         'tools.sessions.on': True,
1326     #         'tools.response_headers.on': True,
1327     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1328     #     }
1329     # }
1330     # cherrypy.Server.ssl_module = 'builtin'
1331     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1332     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1333     # cherrypy.Server.thread_pool = 10
1334     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1335
1336     # cherrypy.config.update({'tools.auth_basic.on': True,
1337     #    'tools.auth_basic.realm': 'localhost',
1338     #    'tools.auth_basic.checkpassword': validate_password})
1339     nbi_server = Server()
1340     cherrypy.engine.subscribe('start', _start_service)
1341     cherrypy.engine.subscribe('stop', _stop_service)
1342     cherrypy.quickstart(nbi_server, '/osm', config_file)
1343
1344
1345 def usage():
1346     print("""Usage: {} [options]
1347         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1348         -h|--help: shows this help
1349         """.format(sys.argv[0]))
1350     # --log-socket-host HOST: send logs to this host")
1351     # --log-socket-port PORT: send logs using this port (default: 9022)")
1352
1353
1354 if __name__ == '__main__':
1355     try:
1356         # load parameters and configuration
1357         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1358         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1359         config_file = None
1360         for o, a in opts:
1361             if o in ("-h", "--help"):
1362                 usage()
1363                 sys.exit()
1364             elif o in ("-c", "--config"):
1365                 config_file = a
1366             # elif o == "--log-socket-port":
1367             #     log_socket_port = a
1368             # elif o == "--log-socket-host":
1369             #     log_socket_host = a
1370             # elif o == "--log-file":
1371             #     log_file = a
1372             else:
1373                 assert False, "Unhandled option"
1374         if config_file:
1375             if not path.isfile(config_file):
1376                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1377                 exit(1)
1378         else:
1379             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1380                 if path.isfile(config_file):
1381                     break
1382             else:
1383                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1384                 exit(1)
1385         nbi(config_file)
1386     except getopt.GetoptError as e:
1387         print(str(e), file=sys.stderr)
1388         # usage()
1389         exit(1)