osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50
  51 """
  52 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  53 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  54         /nsd/v1
  55             /ns_descriptors_content                             O       O
  56                 /<nsdInfoId>                                    O       O       O       O
  57             /ns_descriptors                                     O5      O5
  58                 /<nsdInfoId>                                    O5                      O5      5
  59                     /nsd_content                                O5              O5
  60                     /nsd                                        O
  61                     /artifacts[/<artifactPath>]                 O
  62             /pnf_descriptors                                    5       5
  63                 /<pnfdInfoId>                                   5                       5       5
  64                     /pnfd_content                               5               5
  65             /subscriptions                                      5       5
  66                 /<subscriptionId>                               5                       X
  67
  68         /vnfpkgm/v1
  69             /vnf_packages_content                               O       O
  70                 /<vnfPkgId>                                     O                       O
  71             /vnf_packages                                       O5      O5
  72                 /<vnfPkgId>                                     O5                      O5      5
  73                     /package_content                            O5               O5
  74                         /upload_from_uri                                X
  75                     /vnfd                                       O5
  76                     /artifacts[/<artifactPath>]                 O5
  77             /subscriptions                                      X       X
  78                 /<subscriptionId>                               X                       X
  79
  80         /nslcm/v1
  81             /ns_instances_content                               O       O
  82                 /<nsInstanceId>                                 O                       O
  83             /ns_instances                                       5       5
  84                 /<nsInstanceId>                                 O5                      O5
  85                     instantiate                                         O5
  86                     terminate                                           O5
  87                     action                                              O
  88                     scale                                               O5
  89                     heal                                                5
  90             /ns_lcm_op_occs                                     5       5
  91                 /<nsLcmOpOccId>                                 5                       5       5
  92                     TO BE COMPLETED                             5               5
  93             /vnf_instances  (also vnfrs for compatibility)      O
  94                 /<vnfInstanceId>                                O
  95             /subscriptions                                      5       5
  96                 /<subscriptionId>                               5                       X
  97
  98         /pdu/v1
  99             /pdu_descriptors                                    O       O
 100                 /<id>                                           O               O       O       O
 101
 102         /admin/v1
 103             /tokens                                             O       O
 104                 /<id>                                           O                       O
 105             /users                                              O       O
 106                 /<id>                                           O               O       O       O
 107             /projects                                           O       O
 108                 /<id>                                           O                       O
 109             /vim_accounts  (also vims for compatibility)        O       O
 110                 /<id>                                           O                       O       O
 111             /wim_accounts                                       O       O
 112                 /<id>                                           O                       O       O
 113             /sdns                                               O       O
 114                 /<id>                                           O                       O       O
 115             /k8sclusters                                        O       O
 116                 /<id>                                           O                       O       O
 117             /k8srepos                                           O       O
 118                 /<id>                                           O                               O
 119
 120         /nst/v1                                                 O       O
 121             /netslice_templates_content                         O       O
 122                 /<nstInfoId>                                    O       O       O       O
 123             /netslice_templates                                 O       O
 124                 /<nstInfoId>                                    O                       O       O
 125                     /nst_content                                O               O
 126                     /nst                                        O
 127                     /artifacts[/<artifactPath>]                 O
 128             /subscriptions                                      X       X
 129                 /<subscriptionId>                               X                       X
 130
 131         /nsilcm/v1
 132             /netslice_instances_content                         O       O
 133                 /<SliceInstanceId>                              O                       O
 134             /netslice_instances                                 O       O
 135                 /<SliceInstanceId>                              O                       O
 136                     instantiate                                         O
 137                     terminate                                           O
 138                     action                                              O
 139             /nsi_lcm_op_occs                                    O       O
 140                 /<nsiLcmOpOccId>                                O                       O       O
 141             /subscriptions                                      X       X
 142                 /<subscriptionId>                               X                       X
 143
 144 query string:
 145     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 146         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 147         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 148         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 149         attrName := string
 150     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 151     item of the array, that is, pass if any item of the array pass the filter.
 152     It allows both ne and neq for not equal
 153     TODO: 4.3.3 Attribute selectors
 154         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 155         (none)  … same as “exclude_default”
 156         all_fields      … all attributes.
 157         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 158         conditionally mandatory, and that are not provided in <list>.
 159         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 160         are not conditionally mandatory, and that are provided in <list>.
 161         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 162         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 163         the particular resource
 164         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 165         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 166         present specification for the particular resource, but that are not part of <list>
 167     Additionally it admits some administrator values:
 168         FORCE: To force operations skipping dependency checkings
 169         ADMIN: To act as an administrator or a different project
 170         PUBLIC: To get public descriptors or set a descriptor as public
 171         SET_PROJECT: To make a descriptor available for other project
 172
 173 Header field name       Reference       Example Descriptions
 174     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 175     This header field shall be present if the response is expected to have a non-empty message body.
 176     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 177     This header field shall be present if the request has a non-empty message body.
 178     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 179     Details are specified in clause 4.5.3.
 180     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 181 Header field name       Reference       Example Descriptions
 182     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 183     This header field shall be present if the response has a non-empty message body.
 184     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 185     new resource has been created.
 186     This header field shall be present if the response status code is 201 or 3xx.
 187     In the present document this header field is also used if the response status code is 202 and a new resource was
 188     created.
 189     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 190     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 191     token.
 192     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 193     certain resources.
 194     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 195     response, and the total length of the file.
 196     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 197 """
 198
 199 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 200 # ^ Contains possible administrative query string words:
 201 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 202 #           (not owned by my session project).
 203 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 204 #     FORCE=True(by default)|False: Force edition/deletion operations
 205 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 206
 207 valid_url_methods = {
 208     # contains allowed URL and methods, and the role_permission name
 209     "admin": {
 210         "v1": {
 211             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 212                        "ROLE_PERMISSION": "tokens:",
 213                        "<ID>": {"METHODS": ("GET", "DELETE"),
 214                                 "ROLE_PERMISSION": "tokens:id:"
 215                                 }
 216                        },
 217             "users": {"METHODS": ("GET", "POST"),
 218                       "ROLE_PERMISSION": "users:",
 219                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 220                                "ROLE_PERMISSION": "users:id:"
 221                                }
 222                       },
 223             "projects": {"METHODS": ("GET", "POST"),
 224                          "ROLE_PERMISSION": "projects:",
 225                          "<ID>": {"METHODS": ("GET", "DELETE", "PUT"),
 226                                   "ROLE_PERMISSION": "projects:id:"}
 227                          },
 228             "roles": {"METHODS": ("GET", "POST"),
 229                       "ROLE_PERMISSION": "roles:",
 230                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PUT"),
 231                                "ROLE_PERMISSION": "roles:id:"
 232                                }
 233                       },
 234             "vims": {"METHODS": ("GET", "POST"),
 235                      "ROLE_PERMISSION": "vims:",
 236                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 237                               "ROLE_PERMISSION": "vims:id:"
 238                               }
 239                      },
 240             "vim_accounts": {"METHODS": ("GET", "POST"),
 241                              "ROLE_PERMISSION": "vim_accounts:",
 242                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 243                                       "ROLE_PERMISSION": "vim_accounts:id:"
 244                                       }
 245                              },
 246             "wim_accounts": {"METHODS": ("GET", "POST"),
 247                              "ROLE_PERMISSION": "wim_accounts:",
 248                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 249                                       "ROLE_PERMISSION": "wim_accounts:id:"
 250                                       }
 251                              },
 252             "sdns": {"METHODS": ("GET", "POST"),
 253                      "ROLE_PERMISSION": "sdn_controllers:",
 254                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 255                               "ROLE_PERMISSION": "sdn_controllers:id:"
 256                               }
 257                      },
 258             "k8sclusters": {"METHODS": ("GET", "POST"),
 259                             "ROLE_PERMISSION": "k8sclusters:",
 260                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 261                                      "ROLE_PERMISSION": "k8sclusters:id:"
 262                                      }
 263                             },
 264             "k8srepos": {"METHODS": ("GET", "POST"),
 265                          "ROLE_PERMISSION": "k8srepos:",
 266                          "<ID>": {"METHODS": ("GET", "DELETE"),
 267                                   "ROLE_PERMISSION": "k8srepos:id:"
 268                                   }
 269                          },
 270
 271         }
 272     },
 273     "pdu": {
 274         "v1": {
 275             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 276                                 "ROLE_PERMISSION": "pduds:",
 277                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 278                                          "ROLE_PERMISSION": "pduds:id:"
 279                                          }
 280                                 },
 281         }
 282     },
 283     "nsd": {
 284         "v1": {
 285             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 286                                        "ROLE_PERMISSION": "nsds:",
 287                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 288                                                 "ROLE_PERMISSION": "nsds:id:"
 289                                                 }
 290                                        },
 291             "ns_descriptors": {"METHODS": ("GET", "POST"),
 292                                "ROLE_PERMISSION": "nsds:",
 293                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 294                                         "ROLE_PERMISSION": "nsds:id:",
 295                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 296                                                         "ROLE_PERMISSION": "nsds:id:content:",
 297                                                         },
 298                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 299                                                 "ROLE_PERMISSION": "nsds:id:content:"
 300                                                 },
 301                                         "artifacts": {"*": {"METHODS": ("GET",),
 302                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 303                                                             }
 304                                                       }
 305                                         }
 306                                },
 307             "pnf_descriptors": {"TODO": ("GET", "POST"),
 308                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 309                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 310                                          }
 311                                 },
 312             "subscriptions": {"TODO": ("GET", "POST"),
 313                               "<ID>": {"TODO": ("GET", "DELETE")}
 314                               },
 315         }
 316     },
 317     "vnfpkgm": {
 318         "v1": {
 319             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 320                                      "ROLE_PERMISSION": "vnfds:",
 321                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 322                                               "ROLE_PERMISSION": "vnfds:id:"}
 323                                      },
 324             "vnf_packages": {"METHODS": ("GET", "POST"),
 325                              "ROLE_PERMISSION": "vnfds:",
 326                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 327                                       "ROLE_PERMISSION": "vnfds:id:",
 328                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 329                                                           "ROLE_PERMISSION": "vnfds:id:",
 330                                                           "upload_from_uri": {"METHODS": (),
 331                                                                               "TODO": ("POST", ),
 332                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 333                                                                               }
 334                                                           },
 335                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 336                                                "ROLE_PERMISSION": "vnfds:id:content:"
 337                                                },
 338                                       "artifacts": {"*": {"METHODS": ("GET", ),
 339                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 340                                                           }
 341                                                     }
 342                                       }
 343                              },
 344             "subscriptions": {"TODO": ("GET", "POST"),
 345                               "<ID>": {"TODO": ("GET", "DELETE")}
 346                               },
 347         }
 348     },
 349     "nslcm": {
 350         "v1": {
 351             "ns_instances_content": {"METHODS": ("GET", "POST"),
 352                                      "ROLE_PERMISSION": "ns_instances:",
 353                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 354                                               "ROLE_PERMISSION": "ns_instances:id:"
 355                                               }
 356                                      },
 357             "ns_instances": {"METHODS": ("GET", "POST"),
 358                              "ROLE_PERMISSION": "ns_instances:",
 359                              "<ID>": {"METHODS": ("GET", "DELETE"),
 360                                       "ROLE_PERMISSION": "ns_instances:id:",
 361                                       "scale": {"METHODS": ("POST",),
 362                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 363                                                 },
 364                                       "terminate": {"METHODS": ("POST",),
 365                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 366                                                     },
 367                                       "instantiate": {"METHODS": ("POST",),
 368                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 369                                                       },
 370                                       "action": {"METHODS": ("POST",),
 371                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 372                                                  },
 373                                       }
 374                              },
 375             "ns_lcm_op_occs": {"METHODS": ("GET",),
 376                                "ROLE_PERMISSION": "ns_instances:opps:",
 377                                "<ID>": {"METHODS": ("GET",),
 378                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 379                                         },
 380                                },
 381             "vnfrs": {"METHODS": ("GET",),
 382                       "ROLE_PERMISSION": "vnf_instances:",
 383                       "<ID>": {"METHODS": ("GET",),
 384                                "ROLE_PERMISSION": "vnf_instances:id:"
 385                                }
 386                       },
 387             "vnf_instances": {"METHODS": ("GET",),
 388                               "ROLE_PERMISSION": "vnf_instances:",
 389                               "<ID>": {"METHODS": ("GET",),
 390                                        "ROLE_PERMISSION": "vnf_instances:id:"
 391                                        }
 392                               },
 393         }
 394     },
 395     "nst": {
 396         "v1": {
 397             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 398                                            "ROLE_PERMISSION": "slice_templates:",
 399                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 400                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 401                                            },
 402             "netslice_templates": {"METHODS": ("GET", "POST"),
 403                                    "ROLE_PERMISSION": "slice_templates:",
 404                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 405                                             "TODO": ("PATCH",),
 406                                             "ROLE_PERMISSION": "slice_templates:id:",
 407                                             "nst_content": {"METHODS": ("GET", "PUT"),
 408                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 409                                                             },
 410                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 411                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 412                                                     },
 413                                             "artifacts": {"*": {"METHODS": ("GET",),
 414                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 415                                                                 }
 416                                                           }
 417                                             }
 418                                    },
 419             "subscriptions": {"TODO": ("GET", "POST"),
 420                               "<ID>": {"TODO": ("GET", "DELETE")}
 421                               },
 422         }
 423     },
 424     "nsilcm": {
 425         "v1": {
 426             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 427                                            "ROLE_PERMISSION": "slice_instances:",
 428                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 429                                                     "ROLE_PERMISSION": "slice_instances:id:"
 430                                                     }
 431                                            },
 432             "netslice_instances": {"METHODS": ("GET", "POST"),
 433                                    "ROLE_PERMISSION": "slice_instances:",
 434                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 435                                             "ROLE_PERMISSION": "slice_instances:id:",
 436                                             "terminate": {"METHODS": ("POST",),
 437                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 438                                                           },
 439                                             "instantiate": {"METHODS": ("POST",),
 440                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 441                                                             },
 442                                             "action": {"METHODS": ("POST",),
 443                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 444                                                        },
 445                                             }
 446                                    },
 447             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 448                                 "ROLE_PERMISSION": "slice_instances:opps:",
 449                                 "<ID>": {"METHODS": ("GET",),
 450                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 451                                          },
 452                                 },
 453         }
 454     },
 455     "nspm": {
 456         "v1": {
 457             "pm_jobs": {
 458                 "<ID>": {
 459                     "reports": {
 460                         "<ID>": {"METHODS": ("GET",),
 461                                  "ROLE_PERMISSION": "reports:id:",
 462                                  }
 463                     }
 464                 },
 465             },
 466         },
 467     },
 468 }
 469
 470
 471 class NbiException(Exception):
 472
 473     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 474         Exception.__init__(self, message)
 475         self.http_code = http_code
 476
 477
 478 class Server(object):
 479     instance = 0
 480     # to decode bytes to str
 481     reader = getreader("utf-8")
 482
 483     def __init__(self):
 484         self.instance += 1
 485         self.engine = Engine()
 486         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 487
 488     def _format_in(self, kwargs):
 489         try:
 490             indata = None
 491             if cherrypy.request.body.length:
 492                 error_text = "Invalid input format "
 493
 494                 if "Content-Type" in cherrypy.request.headers:
 495                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 496                         error_text = "Invalid json format "
 497                         indata = json.load(self.reader(cherrypy.request.body))
 498                         cherrypy.request.headers.pop("Content-File-MD5", None)
 499                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 500                         error_text = "Invalid yaml format "
 501                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 502                         cherrypy.request.headers.pop("Content-File-MD5", None)
 503                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 504                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 505                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 506                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 507                         indata = cherrypy.request.body  # .read()
 508                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 509                         if "descriptor_file" in kwargs:
 510                             filecontent = kwargs.pop("descriptor_file")
 511                             if not filecontent.file:
 512                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 513                             indata = filecontent.file  # .read()
 514                             if filecontent.content_type.value:
 515                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 516                     else:
 517                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 518                         #                          "Only 'Content-Type' of type 'application/json' or
 519                         # 'application/yaml' for input format are available")
 520                         error_text = "Invalid yaml format "
 521                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 522                         cherrypy.request.headers.pop("Content-File-MD5", None)
 523                 else:
 524                     error_text = "Invalid yaml format "
 525                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 526                     cherrypy.request.headers.pop("Content-File-MD5", None)
 527             if not indata:
 528                 indata = {}
 529
 530             format_yaml = False
 531             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 532                 format_yaml = True
 533
 534             for k, v in kwargs.items():
 535                 if isinstance(v, str):
 536                     if v == "":
 537                         kwargs[k] = None
 538                     elif format_yaml:
 539                         try:
 540                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 541                         except Exception:
 542                             pass
 543                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 544                         try:
 545                             kwargs[k] = int(v)
 546                         except Exception:
 547                             try:
 548                                 kwargs[k] = float(v)
 549                             except Exception:
 550                                 pass
 551                     elif v.find(",") > 0:
 552                         kwargs[k] = v.split(",")
 553                 elif isinstance(v, (list, tuple)):
 554                     for index in range(0, len(v)):
 555                         if v[index] == "":
 556                             v[index] = None
 557                         elif format_yaml:
 558                             try:
 559                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 560                             except Exception:
 561                                 pass
 562
 563             return indata
 564         except (ValueError, yaml.YAMLError) as exc:
 565             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 566         except KeyError as exc:
 567             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 568         except Exception as exc:
 569             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 570
 571     @staticmethod
 572     def _format_out(data, token_info=None, _format=None):
 573         """
 574         return string of dictionary data according to requested json, yaml, xml. By default json
 575         :param data: response to be sent. Can be a dict, text or file
 576         :param token_info: Contains among other username and project
 577         :param _format: The format to be set as Content-Type if data is a file
 578         :return: None
 579         """
 580         accept = cherrypy.request.headers.get("Accept")
 581         if data is None:
 582             if accept and "text/html" in accept:
 583                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 584             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 585             return
 586         elif hasattr(data, "read"):  # file object
 587             if _format:
 588                 cherrypy.response.headers["Content-Type"] = _format
 589             elif "b" in data.mode:  # binariy asssumig zip
 590                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 591             else:
 592                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 593             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 594             return data
 595         if accept:
 596             if "application/json" in accept:
 597                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 598                 a = json.dumps(data, indent=4) + "\n"
 599                 return a.encode("utf8")
 600             elif "text/html" in accept:
 601                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 602
 603             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 604                 pass
 605             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 606             elif cherrypy.response.status >= 400:
 607                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 608                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 609                                          "for output format are available")
 610         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 611         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 612                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 613
 614     @cherrypy.expose
 615     def index(self, *args, **kwargs):
 616         token_info = None
 617         try:
 618             if cherrypy.request.method == "GET":
 619                 token_info = self.authenticator.authorize()
 620                 outdata = token_info   # Home page
 621             else:
 622                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 623                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 624
 625             return self._format_out(outdata, token_info)
 626
 627         except (EngineException, AuthException) as e:
 628             # cherrypy.log("index Exception {}".format(e))
 629             cherrypy.response.status = e.http_code.value
 630             return self._format_out("Welcome to OSM!", token_info)
 631
 632     @cherrypy.expose
 633     def version(self, *args, **kwargs):
 634         # TODO consider to remove and provide version using the static version file
 635         try:
 636             if cherrypy.request.method != "GET":
 637                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 638             elif args or kwargs:
 639                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 640             # TODO include version of other modules, pick up from some kafka admin message
 641             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 642             return self._format_out(osm_nbi_version)
 643         except NbiException as e:
 644             cherrypy.response.status = e.http_code.value
 645             problem_details = {
 646                 "code": e.http_code.name,
 647                 "status": e.http_code.value,
 648                 "detail": str(e),
 649             }
 650             return self._format_out(problem_details, None)
 651
 652     @staticmethod
 653     def _format_login(token_info):
 654         """
 655         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 656         log this information
 657         :param token_info: Dictionary with token content
 658         :return: None
 659         """
 660         cherrypy.request.login = token_info.get("username", "-")
 661         if token_info.get("project_name"):
 662             cherrypy.request.login += "/" + token_info["project_name"]
 663         if token_info.get("id"):
 664             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 665
 666     @cherrypy.expose
 667     def token(self, method, token_id=None, kwargs=None):
 668         token_info = None
 669         # self.engine.load_dbase(cherrypy.request.app.config)
 670         indata = self._format_in(kwargs)
 671         if not isinstance(indata, dict):
 672             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 673
 674         if method == "GET":
 675             token_info = self.authenticator.authorize()
 676             # for logging
 677             self._format_login(token_info)
 678             if token_id:
 679                 outdata = self.authenticator.get_token(token_info, token_id)
 680             else:
 681                 outdata = self.authenticator.get_token_list(token_info)
 682         elif method == "POST":
 683             try:
 684                 token_info = self.authenticator.authorize()
 685             except Exception:
 686                 token_info = None
 687             if kwargs:
 688                 indata.update(kwargs)
 689             # This is needed to log the user when authentication fails
 690             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 691             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 692             cherrypy.session['Authorization'] = outdata["_id"]
 693             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 694             # for logging
 695             self._format_login(token_info)
 696
 697             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 698             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 699         elif method == "DELETE":
 700             if not token_id and "id" in kwargs:
 701                 token_id = kwargs["id"]
 702             elif not token_id:
 703                 token_info = self.authenticator.authorize()
 704                 # for logging
 705                 self._format_login(token_info)
 706                 token_id = token_info["_id"]
 707             outdata = self.authenticator.del_token(token_id)
 708             token_info = None
 709             cherrypy.session['Authorization'] = "logout"
 710             # cherrypy.response.cookie["Authorization"] = token_id
 711             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 712         else:
 713             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 714         return self._format_out(outdata, token_info)
 715
 716     @cherrypy.expose
 717     def test(self, *args, **kwargs):
 718         thread_info = None
 719         if args and args[0] == "help":
 720             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 721                    "sleep/<time>\nmessage/topic\n</pre></html>"
 722
 723         elif args and args[0] == "init":
 724             try:
 725                 # self.engine.load_dbase(cherrypy.request.app.config)
 726                 self.engine.create_admin()
 727                 return "Done. User 'admin', password 'admin' created"
 728             except Exception:
 729                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 730                 return self._format_out("Database already initialized")
 731         elif args and args[0] == "file":
 732             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 733                                                   "text/plain", "attachment")
 734         elif args and args[0] == "file2":
 735             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 736             f = open(f_path, "r")
 737             cherrypy.response.headers["Content-type"] = "text/plain"
 738             return f
 739
 740         elif len(args) == 2 and args[0] == "db-clear":
 741             deleted_info = self.engine.db.del_list(args[1], kwargs)
 742             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 743         elif len(args) and args[0] == "fs-clear":
 744             if len(args) >= 2:
 745                 folders = (args[1],)
 746             else:
 747                 folders = self.engine.fs.dir_ls(".")
 748             for folder in folders:
 749                 self.engine.fs.file_delete(folder)
 750             return ",".join(folders) + " folders deleted\n"
 751         elif args and args[0] == "login":
 752             if not cherrypy.request.headers.get("Authorization"):
 753                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 754                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 755         elif args and args[0] == "login2":
 756             if not cherrypy.request.headers.get("Authorization"):
 757                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 758                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 759         elif args and args[0] == "sleep":
 760             sleep_time = 5
 761             try:
 762                 sleep_time = int(args[1])
 763             except Exception:
 764                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 765                 return self._format_out("Database already initialized")
 766             thread_info = cherrypy.thread_data
 767             print(thread_info)
 768             time.sleep(sleep_time)
 769             # thread_info
 770         elif len(args) >= 2 and args[0] == "message":
 771             main_topic = args[1]
 772             return_text = "<html><pre>{} ->\n".format(main_topic)
 773             try:
 774                 if cherrypy.request.method == 'POST':
 775                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 776                     for k, v in to_send.items():
 777                         self.engine.msg.write(main_topic, k, v)
 778                         return_text += "  {}: {}\n".format(k, v)
 779                 elif cherrypy.request.method == 'GET':
 780                     for k, v in kwargs.items():
 781                         self.engine.msg.write(main_topic, k, yaml.load(v), Loader=yaml.SafeLoader)
 782                         return_text += "  {}: {}\n".format(k, yaml.load(v), Loader=yaml.SafeLoader)
 783             except Exception as e:
 784                 return_text += "Error: " + str(e)
 785             return_text += "</pre></html>\n"
 786             return return_text
 787
 788         return_text = (
 789             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 790             "  kwargs: {}\n".format(kwargs) +
 791             "  headers: {}\n".format(cherrypy.request.headers) +
 792             "  path_info: {}\n".format(cherrypy.request.path_info) +
 793             "  query_string: {}\n".format(cherrypy.request.query_string) +
 794             "  session: {}\n".format(cherrypy.session) +
 795             "  cookie: {}\n".format(cherrypy.request.cookie) +
 796             "  method: {}\n".format(cherrypy.request.method) +
 797             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 798             "  body:\n")
 799         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 800         if cherrypy.request.body.length:
 801             return_text += "    content: {}\n".format(
 802                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 803         if thread_info:
 804             return_text += "thread: {}\n".format(thread_info)
 805         return_text += "</pre></html>"
 806         return return_text
 807
 808     @staticmethod
 809     def _check_valid_url_method(method, *args):
 810         if len(args) < 3:
 811             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 812
 813         reference = valid_url_methods
 814         for arg in args:
 815             if arg is None:
 816                 break
 817             if not isinstance(reference, dict):
 818                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 819                                    HTTPStatus.METHOD_NOT_ALLOWED)
 820
 821             if arg in reference:
 822                 reference = reference[arg]
 823             elif "<ID>" in reference:
 824                 reference = reference["<ID>"]
 825             elif "*" in reference:
 826                 reference = reference["*"]
 827                 break
 828             else:
 829                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 830         if "TODO" in reference and method in reference["TODO"]:
 831             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 832         elif "METHODS" in reference and method not in reference["METHODS"]:
 833             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 834         return reference["ROLE_PERMISSION"] + method.lower()
 835
 836     @staticmethod
 837     def _set_location_header(main_topic, version, topic, id):
 838         """
 839         Insert response header Location with the URL of created item base on URL params
 840         :param main_topic:
 841         :param version:
 842         :param topic:
 843         :param id:
 844         :return: None
 845         """
 846         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 847         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 848         return
 849
 850     @staticmethod
 851     def _extract_query_string_operations(kwargs, method):
 852         """
 853
 854         :param kwargs:
 855         :return:
 856         """
 857         query_string_operations = []
 858         if kwargs:
 859             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 860                 if qs in kwargs and kwargs[qs].lower() != "false":
 861                     query_string_operations.append(qs.lower() + ":" + method.lower())
 862         return query_string_operations
 863
 864     @staticmethod
 865     def _manage_admin_query(token_info, kwargs, method, _id):
 866         """
 867         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 868         Check that users has rights to use them and returs the admin_query
 869         :param token_info: token_info rights obtained by token
 870         :param kwargs: query string input.
 871         :param method: http method: GET, POSST, PUT, ...
 872         :param _id:
 873         :return: admin_query dictionary with keys:
 874             public: True, False or None
 875             force: True or False
 876             project_id: tuple with projects used for accessing an element
 877             set_project: tuple with projects that a created element will belong to
 878             method: show, list, delete, write
 879         """
 880         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 881                        "admin": token_info["admin"], "public": None,
 882                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 883         if kwargs:
 884             # FORCE
 885             if "FORCE" in kwargs:
 886                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 887                     admin_query["force"] = True
 888                 del kwargs["FORCE"]
 889             # PUBLIC
 890             if "PUBLIC" in kwargs:
 891                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 892                     admin_query["public"] = True
 893                 else:
 894                     admin_query["public"] = False
 895                 del kwargs["PUBLIC"]
 896             # ADMIN
 897             if "ADMIN" in kwargs:
 898                 behave_as = kwargs.pop("ADMIN")
 899                 if behave_as.lower() != "false":
 900                     if not token_info["admin"]:
 901                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 902                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 903                         admin_query["project_id"] = ()
 904                     elif isinstance(behave_as, (list, tuple)):
 905                         admin_query["project_id"] = behave_as
 906                     else:   # isinstance(behave_as, str)
 907                         admin_query["project_id"] = (behave_as, )
 908             if "SET_PROJECT" in kwargs:
 909                 set_project = kwargs.pop("SET_PROJECT")
 910                 if not set_project:
 911                     admin_query["set_project"] = list(admin_query["project_id"])
 912                 else:
 913                     if isinstance(set_project, str):
 914                         set_project = (set_project, )
 915                     if admin_query["project_id"]:
 916                         for p in set_project:
 917                             if p not in admin_query["project_id"]:
 918                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 919                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 920                     admin_query["set_project"] = set_project
 921
 922             # PROJECT_READ
 923             # if "PROJECT_READ" in kwargs:
 924             #     admin_query["project"] = kwargs.pop("project")
 925             #     if admin_query["project"] == token_info["project_id"]:
 926         if method == "GET":
 927             if _id:
 928                 admin_query["method"] = "show"
 929             else:
 930                 admin_query["method"] = "list"
 931         elif method == "DELETE":
 932             admin_query["method"] = "delete"
 933         else:
 934             admin_query["method"] = "write"
 935         return admin_query
 936
 937     @cherrypy.expose
 938     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 939         token_info = None
 940         outdata = None
 941         _format = None
 942         method = "DONE"
 943         engine_topic = None
 944         rollback = []
 945         engine_session = None
 946         try:
 947             if not main_topic or not version or not topic:
 948                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 949                                    HTTPStatus.METHOD_NOT_ALLOWED)
 950             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 951                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 952                                    HTTPStatus.METHOD_NOT_ALLOWED)
 953             if version != 'v1':
 954                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 955
 956             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 957                 method = kwargs.pop("METHOD")
 958             else:
 959                 method = cherrypy.request.method
 960
 961             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 962             query_string_operations = self._extract_query_string_operations(kwargs, method)
 963             if main_topic == "admin" and topic == "tokens":
 964                 return self.token(method, _id, kwargs)
 965             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
 966             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 967             indata = self._format_in(kwargs)
 968             engine_topic = topic
 969             if topic == "subscriptions":
 970                 engine_topic = main_topic + "_" + topic
 971             if item and topic != "pm_jobs":
 972                 engine_topic = item
 973
 974             if main_topic == "nsd":
 975                 engine_topic = "nsds"
 976             elif main_topic == "vnfpkgm":
 977                 engine_topic = "vnfds"
 978             elif main_topic == "nslcm":
 979                 engine_topic = "nsrs"
 980                 if topic == "ns_lcm_op_occs":
 981                     engine_topic = "nslcmops"
 982                 if topic == "vnfrs" or topic == "vnf_instances":
 983                     engine_topic = "vnfrs"
 984             elif main_topic == "nst":
 985                 engine_topic = "nsts"
 986             elif main_topic == "nsilcm":
 987                 engine_topic = "nsis"
 988                 if topic == "nsi_lcm_op_occs":
 989                     engine_topic = "nsilcmops"
 990             elif main_topic == "pdu":
 991                 engine_topic = "pdus"
 992             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
 993                 engine_topic = "vim_accounts"
 994
 995             if method == "GET":
 996                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
 997                     if item in ("vnfd", "nsd", "nst"):
 998                         path = "$DESCRIPTOR"
 999                     elif args:
1000                         path = args
1001                     elif item == "artifacts":
1002                         path = ()
1003                     else:
1004                         path = None
1005                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1006                                                          cherrypy.request.headers.get("Accept"))
1007                     outdata = file
1008                 elif not _id:
1009                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1010                 else:
1011                     if item == "reports":
1012                         # TODO check that project_id (_id in this context) has permissions
1013                         _id = args[0]
1014                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1015             elif method == "POST":
1016                 cherrypy.response.status = HTTPStatus.CREATED.value
1017                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1018                     _id = cherrypy.request.headers.get("Transaction-Id")
1019                     if not _id:
1020                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1021                                                       cherrypy.request.headers)
1022                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1023                                                            cherrypy.request.headers)
1024                     if completed:
1025                         self._set_location_header(main_topic, version, topic, _id)
1026                     else:
1027                         cherrypy.response.headers["Transaction-Id"] = _id
1028                     outdata = {"id": _id}
1029                 elif topic == "ns_instances_content":
1030                     # creates NSR
1031                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1032                     # creates nslcmop
1033                     indata["lcmOperationType"] = "instantiate"
1034                     indata["nsInstanceId"] = _id
1035                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1036                     self._set_location_header(main_topic, version, topic, _id)
1037                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1038                 elif topic == "ns_instances" and item:
1039                     indata["lcmOperationType"] = item
1040                     indata["nsInstanceId"] = _id
1041                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1042                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1043                     outdata = {"id": _id}
1044                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1045                 elif topic == "netslice_instances_content":
1046                     # creates NetSlice_Instance_record (NSIR)
1047                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1048                     self._set_location_header(main_topic, version, topic, _id)
1049                     indata["lcmOperationType"] = "instantiate"
1050                     indata["netsliceInstanceId"] = _id
1051                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1052                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1053
1054                 elif topic == "netslice_instances" and item:
1055                     indata["lcmOperationType"] = item
1056                     indata["netsliceInstanceId"] = _id
1057                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1058                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1059                     outdata = {"id": _id}
1060                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1061                 else:
1062                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1063                                                       cherrypy.request.headers)
1064                     self._set_location_header(main_topic, version, topic, _id)
1065                     outdata = {"id": _id}
1066                     if op_id:
1067                         outdata["op_id"] = op_id
1068                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1069                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1070
1071             elif method == "DELETE":
1072                 if not _id:
1073                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1074                     cherrypy.response.status = HTTPStatus.OK.value
1075                 else:  # len(args) > 1
1076                     delete_in_process = False
1077                     if topic == "ns_instances_content" and not engine_session["force"]:
1078                         nslcmop_desc = {
1079                             "lcmOperationType": "terminate",
1080                             "nsInstanceId": _id,
1081                             "autoremove": True
1082                         }
1083                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, None)
1084                         if opp_id:
1085                             delete_in_process = True
1086                             outdata = {"_id": opp_id}
1087                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1088                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1089                         nsilcmop_desc = {
1090                             "lcmOperationType": "terminate",
1091                             "netsliceInstanceId": _id,
1092                             "autoremove": True
1093                         }
1094                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1095                         if opp_id:
1096                             delete_in_process = True
1097                             outdata = {"_id": opp_id}
1098                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1099                     if not delete_in_process:
1100                         self.engine.del_item(engine_session, engine_topic, _id)
1101                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1102                 if engine_topic in ("vim_accounts", "wim_accounts", "sdns", "k8sclusters", "k8srepos"):
1103                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1104
1105             elif method in ("PUT", "PATCH"):
1106                 op_id = None
1107                 if not indata and not kwargs and not engine_session.get("set_project"):
1108                     raise NbiException("Nothing to update. Provide payload and/or query string",
1109                                        HTTPStatus.BAD_REQUEST)
1110                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1111                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1112                                                            cherrypy.request.headers)
1113                     if not completed:
1114                         cherrypy.response.headers["Transaction-Id"] = id
1115                 else:
1116                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1117
1118                 if op_id:
1119                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1120                     outdata = {"op_id": op_id}
1121                 else:
1122                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1123                     outdata = None
1124             else:
1125                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1126
1127             # if Role information changes, it is needed to reload the information of roles
1128             if topic == "roles" and method != "GET":
1129                 self.authenticator.load_operation_to_allowed_roles()
1130             return self._format_out(outdata, token_info, _format)
1131         except Exception as e:
1132             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1133                               ValidationError, AuthconnException)):
1134                 http_code_value = cherrypy.response.status = e.http_code.value
1135                 http_code_name = e.http_code.name
1136                 cherrypy.log("Exception {}".format(e))
1137             else:
1138                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1139                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1140                 http_code_name = HTTPStatus.BAD_REQUEST.name
1141             if hasattr(outdata, "close"):  # is an open file
1142                 outdata.close()
1143             error_text = str(e)
1144             rollback.reverse()
1145             for rollback_item in rollback:
1146                 try:
1147                     if rollback_item.get("operation") == "set":
1148                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1149                                                rollback_item["content"], fail_on_empty=False)
1150                     else:
1151                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1152                                                fail_on_empty=False)
1153                 except Exception as e2:
1154                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1155                     cherrypy.log(rollback_error_text)
1156                     error_text += ". " + rollback_error_text
1157             # if isinstance(e, MsgException):
1158             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1159             #         engine_topic[:-1], method, error_text)
1160             problem_details = {
1161                 "code": http_code_name,
1162                 "status": http_code_value,
1163                 "detail": error_text,
1164             }
1165             return self._format_out(problem_details, token_info)
1166             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1167         finally:
1168             if token_info:
1169                 self._format_login(token_info)
1170                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1171                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1172                         if outdata.get(logging_id):
1173                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1174
1175
1176 def _start_service():
1177     """
1178     Callback function called when cherrypy.engine starts
1179     Override configuration with env variables
1180     Set database, storage, message configuration
1181     Init database with admin/admin user password
1182     """
1183     global nbi_server
1184     global subscription_thread
1185     cherrypy.log.error("Starting osm_nbi")
1186     # update general cherrypy configuration
1187     update_dict = {}
1188
1189     engine_config = cherrypy.tree.apps['/osm'].config
1190     for k, v in environ.items():
1191         if not k.startswith("OSMNBI_"):
1192             continue
1193         k1, _, k2 = k[7:].lower().partition("_")
1194         if not k2:
1195             continue
1196         try:
1197             # update static configuration
1198             if k == 'OSMNBI_STATIC_DIR':
1199                 engine_config["/static"]['tools.staticdir.dir'] = v
1200                 engine_config["/static"]['tools.staticdir.on'] = True
1201             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1202                 update_dict['server.socket_port'] = int(v)
1203             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1204                 update_dict['server.socket_host'] = v
1205             elif k1 in ("server", "test", "auth", "log"):
1206                 update_dict[k1 + '.' + k2] = v
1207             elif k1 in ("message", "database", "storage", "authentication"):
1208                 # k2 = k2.replace('_', '.')
1209                 if k2 in ("port", "db_port"):
1210                     engine_config[k1][k2] = int(v)
1211                 else:
1212                     engine_config[k1][k2] = v
1213
1214         except ValueError as e:
1215             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1216         except Exception as e:
1217             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1218
1219     if update_dict:
1220         cherrypy.config.update(update_dict)
1221         engine_config["global"].update(update_dict)
1222
1223     # logging cherrypy
1224     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1225     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1226     logger_server = logging.getLogger("cherrypy.error")
1227     logger_access = logging.getLogger("cherrypy.access")
1228     logger_cherry = logging.getLogger("cherrypy")
1229     logger_nbi = logging.getLogger("nbi")
1230
1231     if "log.file" in engine_config["global"]:
1232         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1233                                                             maxBytes=100e6, backupCount=9, delay=0)
1234         file_handler.setFormatter(log_formatter_simple)
1235         logger_cherry.addHandler(file_handler)
1236         logger_nbi.addHandler(file_handler)
1237     # log always to standard output
1238     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1239                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1240                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1241                             }.items():
1242         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1243         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1244         str_handler = logging.StreamHandler()
1245         str_handler.setFormatter(log_formatter_cherry)
1246         logger.addHandler(str_handler)
1247
1248     if engine_config["global"].get("log.level"):
1249         logger_cherry.setLevel(engine_config["global"]["log.level"])
1250         logger_nbi.setLevel(engine_config["global"]["log.level"])
1251
1252     # logging other modules
1253     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1254         engine_config[k1]["logger_name"] = logname
1255         logger_module = logging.getLogger(logname)
1256         if "logfile" in engine_config[k1]:
1257             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1258                                                                 maxBytes=100e6, backupCount=9, delay=0)
1259             file_handler.setFormatter(log_formatter_simple)
1260             logger_module.addHandler(file_handler)
1261         if "loglevel" in engine_config[k1]:
1262             logger_module.setLevel(engine_config[k1]["loglevel"])
1263     # TODO add more entries, e.g.: storage
1264     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1265     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1266     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1267     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1268
1269     # start subscriptions thread:
1270     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1271     subscription_thread.start()
1272     # Do not capture except SubscriptionException
1273
1274     # load and print version. Ignore possible errors, e.g. file not found
1275     try:
1276         backend = engine_config["authentication"]["backend"]
1277         nbi_version
1278         cherrypy.log.error("Starting OSM NBI Version '{}' with '{}' authentication backend"
1279                            .format(nbi_version + " " + nbi_version_date, backend))
1280     except Exception:
1281         pass
1282
1283
1284 def _stop_service():
1285     """
1286     Callback function called when cherrypy.engine stops
1287     TODO: Ending database connections.
1288     """
1289     global subscription_thread
1290     if subscription_thread:
1291         subscription_thread.terminate()
1292     subscription_thread = None
1293     cherrypy.tree.apps['/osm'].root.engine.stop()
1294     cherrypy.log.error("Stopping osm_nbi")
1295
1296
1297 def nbi(config_file):
1298     global nbi_server
1299     # conf = {
1300     #     '/': {
1301     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1302     #         'tools.sessions.on': True,
1303     #         'tools.response_headers.on': True,
1304     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1305     #     }
1306     # }
1307     # cherrypy.Server.ssl_module = 'builtin'
1308     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1309     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1310     # cherrypy.Server.thread_pool = 10
1311     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1312
1313     # cherrypy.config.update({'tools.auth_basic.on': True,
1314     #    'tools.auth_basic.realm': 'localhost',
1315     #    'tools.auth_basic.checkpassword': validate_password})
1316     nbi_server = Server()
1317     cherrypy.engine.subscribe('start', _start_service)
1318     cherrypy.engine.subscribe('stop', _stop_service)
1319     cherrypy.quickstart(nbi_server, '/osm', config_file)
1320
1321
1322 def usage():
1323     print("""Usage: {} [options]
1324         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1325         -h|--help: shows this help
1326         """.format(sys.argv[0]))
1327     # --log-socket-host HOST: send logs to this host")
1328     # --log-socket-port PORT: send logs using this port (default: 9022)")
1329
1330
1331 if __name__ == '__main__':
1332     try:
1333         # load parameters and configuration
1334         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1335         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1336         config_file = None
1337         for o, a in opts:
1338             if o in ("-h", "--help"):
1339                 usage()
1340                 sys.exit()
1341             elif o in ("-c", "--config"):
1342                 config_file = a
1343             # elif o == "--log-socket-port":
1344             #     log_socket_port = a
1345             # elif o == "--log-socket-host":
1346             #     log_socket_host = a
1347             # elif o == "--log-file":
1348             #     log_file = a
1349             else:
1350                 assert False, "Unhandled option"
1351         if config_file:
1352             if not path.isfile(config_file):
1353                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1354                 exit(1)
1355         else:
1356             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1357                 if path.isfile(config_file):
1358                     break
1359             else:
1360                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1361                 exit(1)
1362         nbi(config_file)
1363     except getopt.GetoptError as e:
1364         print(str(e), file=sys.stderr)
1365         # usage()
1366         exit(1)