osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118             /osmrepos                                           O       O
 119                 /<id>                                           O                               O
 120
 121         /nst/v1                                                 O       O
 122             /netslice_templates_content                         O       O
 123                 /<nstInfoId>                                    O       O       O       O
 124             /netslice_templates                                 O       O
 125                 /<nstInfoId>                                    O                       O       O
 126                     /nst_content                                O               O
 127                     /nst                                        O
 128                     /artifacts[/<artifactPath>]                 O
 129             /subscriptions                                      X       X
 130                 /<subscriptionId>                               X                       X
 131
 132         /nsilcm/v1
 133             /netslice_instances_content                         O       O
 134                 /<SliceInstanceId>                              O                       O
 135             /netslice_instances                                 O       O
 136                 /<SliceInstanceId>                              O                       O
 137                     instantiate                                         O
 138                     terminate                                           O
 139                     action                                              O
 140             /nsi_lcm_op_occs                                    O       O
 141                 /<nsiLcmOpOccId>                                O                       O       O
 142             /subscriptions                                      X       X
 143                 /<subscriptionId>                               X                       X
 144
 145 query string:
 146     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 147         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 148         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 149         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 150         attrName := string
 151     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 152     item of the array, that is, pass if any item of the array pass the filter.
 153     It allows both ne and neq for not equal
 154     TODO: 4.3.3 Attribute selectors
 155         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 156         (none)  … same as “exclude_default”
 157         all_fields      … all attributes.
 158         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 159         conditionally mandatory, and that are not provided in <list>.
 160         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 161         are not conditionally mandatory, and that are provided in <list>.
 162         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 163         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 164         the particular resource
 165         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 166         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 167         present specification for the particular resource, but that are not part of <list>
 168     Additionally it admits some administrator values:
 169         FORCE: To force operations skipping dependency checkings
 170         ADMIN: To act as an administrator or a different project
 171         PUBLIC: To get public descriptors or set a descriptor as public
 172         SET_PROJECT: To make a descriptor available for other project
 173
 174 Header field name       Reference       Example Descriptions
 175     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 176     This header field shall be present if the response is expected to have a non-empty message body.
 177     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 178     This header field shall be present if the request has a non-empty message body.
 179     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 180     Details are specified in clause 4.5.3.
 181     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 182 Header field name       Reference       Example Descriptions
 183     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 184     This header field shall be present if the response has a non-empty message body.
 185     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 186     new resource has been created.
 187     This header field shall be present if the response status code is 201 or 3xx.
 188     In the present document this header field is also used if the response status code is 202 and a new resource was
 189     created.
 190     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 191     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 192     token.
 193     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 194     certain resources.
 195     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 196     response, and the total length of the file.
 197     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 198 """
 199
 200 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 201 # ^ Contains possible administrative query string words:
 202 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 203 #           (not owned by my session project).
 204 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 205 #     FORCE=True(by default)|False: Force edition/deletion operations
 206 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 207
 208 valid_url_methods = {
 209     # contains allowed URL and methods, and the role_permission name
 210     "admin": {
 211         "v1": {
 212             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 213                        "ROLE_PERMISSION": "tokens:",
 214                        "<ID>": {"METHODS": ("GET", "DELETE"),
 215                                 "ROLE_PERMISSION": "tokens:id:"
 216                                 }
 217                        },
 218             "users": {"METHODS": ("GET", "POST"),
 219                       "ROLE_PERMISSION": "users:",
 220                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 221                                "ROLE_PERMISSION": "users:id:"
 222                                }
 223                       },
 224             "projects": {"METHODS": ("GET", "POST"),
 225                          "ROLE_PERMISSION": "projects:",
 226                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 227                                   "ROLE_PERMISSION": "projects:id:"}
 228                          },
 229             "roles": {"METHODS": ("GET", "POST"),
 230                       "ROLE_PERMISSION": "roles:",
 231                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 232                                "ROLE_PERMISSION": "roles:id:"
 233                                }
 234                       },
 235             "vims": {"METHODS": ("GET", "POST"),
 236                      "ROLE_PERMISSION": "vims:",
 237                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 238                               "ROLE_PERMISSION": "vims:id:"
 239                               }
 240                      },
 241             "vim_accounts": {"METHODS": ("GET", "POST"),
 242                              "ROLE_PERMISSION": "vim_accounts:",
 243                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 244                                       "ROLE_PERMISSION": "vim_accounts:id:"
 245                                       }
 246                              },
 247             "wim_accounts": {"METHODS": ("GET", "POST"),
 248                              "ROLE_PERMISSION": "wim_accounts:",
 249                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 250                                       "ROLE_PERMISSION": "wim_accounts:id:"
 251                                       }
 252                              },
 253             "sdns": {"METHODS": ("GET", "POST"),
 254                      "ROLE_PERMISSION": "sdn_controllers:",
 255                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 256                               "ROLE_PERMISSION": "sdn_controllers:id:"
 257                               }
 258                      },
 259             "k8sclusters": {"METHODS": ("GET", "POST"),
 260                             "ROLE_PERMISSION": "k8sclusters:",
 261                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 262                                      "ROLE_PERMISSION": "k8sclusters:id:"
 263                                      }
 264                             },
 265             "k8srepos": {"METHODS": ("GET", "POST"),
 266                          "ROLE_PERMISSION": "k8srepos:",
 267                          "<ID>": {"METHODS": ("GET", "DELETE"),
 268                                   "ROLE_PERMISSION": "k8srepos:id:"
 269                                   }
 270                          },
 271             "osmrepos": {"METHODS": ("GET", "POST"),
 272                          "ROLE_PERMISSION": "osmrepos:",
 273                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 274                                   "ROLE_PERMISSION": "osmrepos:id:"
 275                                   }
 276                          },
 277             "domains": {"METHODS": ("GET", ),
 278                         "ROLE_PERMISSION": "domains:",
 279                         },
 280         }
 281     },
 282     "pdu": {
 283         "v1": {
 284             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 285                                 "ROLE_PERMISSION": "pduds:",
 286                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 287                                          "ROLE_PERMISSION": "pduds:id:"
 288                                          }
 289                                 },
 290         }
 291     },
 292     "nsd": {
 293         "v1": {
 294             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 295                                        "ROLE_PERMISSION": "nsds:",
 296                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 297                                                 "ROLE_PERMISSION": "nsds:id:"
 298                                                 }
 299                                        },
 300             "ns_descriptors": {"METHODS": ("GET", "POST"),
 301                                "ROLE_PERMISSION": "nsds:",
 302                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 303                                         "ROLE_PERMISSION": "nsds:id:",
 304                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 305                                                         "ROLE_PERMISSION": "nsds:id:content:",
 306                                                         },
 307                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 308                                                 "ROLE_PERMISSION": "nsds:id:content:"
 309                                                 },
 310                                         "artifacts": {"*": {"METHODS": ("GET",),
 311                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 312                                                             }
 313                                                       }
 314                                         }
 315                                },
 316             "pnf_descriptors": {"TODO": ("GET", "POST"),
 317                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 318                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 319                                          }
 320                                 },
 321             "subscriptions": {"TODO": ("GET", "POST"),
 322                               "<ID>": {"TODO": ("GET", "DELETE")}
 323                               },
 324         }
 325     },
 326     "vnfpkgm": {
 327         "v1": {
 328             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 329                                      "ROLE_PERMISSION": "vnfds:",
 330                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 331                                               "ROLE_PERMISSION": "vnfds:id:"}
 332                                      },
 333             "vnf_packages": {"METHODS": ("GET", "POST"),
 334                              "ROLE_PERMISSION": "vnfds:",
 335                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 336                                       "ROLE_PERMISSION": "vnfds:id:",
 337                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 338                                                           "ROLE_PERMISSION": "vnfds:id:",
 339                                                           "upload_from_uri": {"METHODS": (),
 340                                                                               "TODO": ("POST", ),
 341                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 342                                                                               }
 343                                                           },
 344                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 345                                                "ROLE_PERMISSION": "vnfds:id:content:"
 346                                                },
 347                                       "artifacts": {"*": {"METHODS": ("GET", ),
 348                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 349                                                           }
 350                                                     },
 351                                       "action": {"METHODS": ("POST", ),
 352                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 353                                                  },
 354                                       }
 355                              },
 356             "subscriptions": {"TODO": ("GET", "POST"),
 357                               "<ID>": {"TODO": ("GET", "DELETE")}
 358                               },
 359             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 360                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 361                                "<ID>": {"METHODS": ("GET", ),
 362                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 363                                         }
 364                                },
 365         }
 366     },
 367     "nslcm": {
 368         "v1": {
 369             "ns_instances_content": {"METHODS": ("GET", "POST"),
 370                                      "ROLE_PERMISSION": "ns_instances:",
 371                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 372                                               "ROLE_PERMISSION": "ns_instances:id:"
 373                                               }
 374                                      },
 375             "ns_instances": {"METHODS": ("GET", "POST"),
 376                              "ROLE_PERMISSION": "ns_instances:",
 377                              "<ID>": {"METHODS": ("GET", "DELETE"),
 378                                       "ROLE_PERMISSION": "ns_instances:id:",
 379                                       "scale": {"METHODS": ("POST",),
 380                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 381                                                 },
 382                                       "terminate": {"METHODS": ("POST",),
 383                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 384                                                     },
 385                                       "instantiate": {"METHODS": ("POST",),
 386                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 387                                                       },
 388                                       "action": {"METHODS": ("POST",),
 389                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 390                                                  },
 391                                       }
 392                              },
 393             "ns_lcm_op_occs": {"METHODS": ("GET",),
 394                                "ROLE_PERMISSION": "ns_instances:opps:",
 395                                "<ID>": {"METHODS": ("GET",),
 396                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 397                                         },
 398                                },
 399             "vnfrs": {"METHODS": ("GET",),
 400                       "ROLE_PERMISSION": "vnf_instances:",
 401                       "<ID>": {"METHODS": ("GET",),
 402                                "ROLE_PERMISSION": "vnf_instances:id:"
 403                                }
 404                       },
 405             "vnf_instances": {"METHODS": ("GET",),
 406                               "ROLE_PERMISSION": "vnf_instances:",
 407                               "<ID>": {"METHODS": ("GET",),
 408                                        "ROLE_PERMISSION": "vnf_instances:id:"
 409                                        }
 410                               },
 411         }
 412     },
 413     "nst": {
 414         "v1": {
 415             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 416                                            "ROLE_PERMISSION": "slice_templates:",
 417                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 418                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 419                                            },
 420             "netslice_templates": {"METHODS": ("GET", "POST"),
 421                                    "ROLE_PERMISSION": "slice_templates:",
 422                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 423                                             "TODO": ("PATCH",),
 424                                             "ROLE_PERMISSION": "slice_templates:id:",
 425                                             "nst_content": {"METHODS": ("GET", "PUT"),
 426                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 427                                                             },
 428                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 429                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 430                                                     },
 431                                             "artifacts": {"*": {"METHODS": ("GET",),
 432                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 433                                                                 }
 434                                                           }
 435                                             }
 436                                    },
 437             "subscriptions": {"TODO": ("GET", "POST"),
 438                               "<ID>": {"TODO": ("GET", "DELETE")}
 439                               },
 440         }
 441     },
 442     "nsilcm": {
 443         "v1": {
 444             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 445                                            "ROLE_PERMISSION": "slice_instances:",
 446                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 447                                                     "ROLE_PERMISSION": "slice_instances:id:"
 448                                                     }
 449                                            },
 450             "netslice_instances": {"METHODS": ("GET", "POST"),
 451                                    "ROLE_PERMISSION": "slice_instances:",
 452                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 453                                             "ROLE_PERMISSION": "slice_instances:id:",
 454                                             "terminate": {"METHODS": ("POST",),
 455                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 456                                                           },
 457                                             "instantiate": {"METHODS": ("POST",),
 458                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 459                                                             },
 460                                             "action": {"METHODS": ("POST",),
 461                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 462                                                        },
 463                                             }
 464                                    },
 465             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 466                                 "ROLE_PERMISSION": "slice_instances:opps:",
 467                                 "<ID>": {"METHODS": ("GET",),
 468                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 469                                          },
 470                                 },
 471         }
 472     },
 473     "nspm": {
 474         "v1": {
 475             "pm_jobs": {
 476                 "<ID>": {
 477                     "reports": {
 478                         "<ID>": {"METHODS": ("GET",),
 479                                  "ROLE_PERMISSION": "reports:id:",
 480                                  }
 481                     }
 482                 },
 483             },
 484         },
 485     },
 486 }
 487
 488
 489 class NbiException(Exception):
 490
 491     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 492         Exception.__init__(self, message)
 493         self.http_code = http_code
 494
 495
 496 class Server(object):
 497     instance = 0
 498     # to decode bytes to str
 499     reader = getreader("utf-8")
 500
 501     def __init__(self):
 502         self.instance += 1
 503         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 504         self.engine = Engine(self.authenticator)
 505
 506     def _format_in(self, kwargs):
 507         try:
 508             indata = None
 509             if cherrypy.request.body.length:
 510                 error_text = "Invalid input format "
 511
 512                 if "Content-Type" in cherrypy.request.headers:
 513                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 514                         error_text = "Invalid json format "
 515                         indata = json.load(self.reader(cherrypy.request.body))
 516                         cherrypy.request.headers.pop("Content-File-MD5", None)
 517                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 518                         error_text = "Invalid yaml format "
 519                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 520                         cherrypy.request.headers.pop("Content-File-MD5", None)
 521                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 522                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 523                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 524                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 525                         indata = cherrypy.request.body  # .read()
 526                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 527                         if "descriptor_file" in kwargs:
 528                             filecontent = kwargs.pop("descriptor_file")
 529                             if not filecontent.file:
 530                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 531                             indata = filecontent.file  # .read()
 532                             if filecontent.content_type.value:
 533                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 534                     else:
 535                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 536                         #                          "Only 'Content-Type' of type 'application/json' or
 537                         # 'application/yaml' for input format are available")
 538                         error_text = "Invalid yaml format "
 539                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 540                         cherrypy.request.headers.pop("Content-File-MD5", None)
 541                 else:
 542                     error_text = "Invalid yaml format "
 543                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 544                     cherrypy.request.headers.pop("Content-File-MD5", None)
 545             if not indata:
 546                 indata = {}
 547
 548             format_yaml = False
 549             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 550                 format_yaml = True
 551
 552             for k, v in kwargs.items():
 553                 if isinstance(v, str):
 554                     if v == "":
 555                         kwargs[k] = None
 556                     elif format_yaml:
 557                         try:
 558                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 559                         except Exception:
 560                             pass
 561                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 562                         try:
 563                             kwargs[k] = int(v)
 564                         except Exception:
 565                             try:
 566                                 kwargs[k] = float(v)
 567                             except Exception:
 568                                 pass
 569                     elif v.find(",") > 0:
 570                         kwargs[k] = v.split(",")
 571                 elif isinstance(v, (list, tuple)):
 572                     for index in range(0, len(v)):
 573                         if v[index] == "":
 574                             v[index] = None
 575                         elif format_yaml:
 576                             try:
 577                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 578                             except Exception:
 579                                 pass
 580
 581             return indata
 582         except (ValueError, yaml.YAMLError) as exc:
 583             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 584         except KeyError as exc:
 585             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 586         except Exception as exc:
 587             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 588
 589     @staticmethod
 590     def _format_out(data, token_info=None, _format=None):
 591         """
 592         return string of dictionary data according to requested json, yaml, xml. By default json
 593         :param data: response to be sent. Can be a dict, text or file
 594         :param token_info: Contains among other username and project
 595         :param _format: The format to be set as Content-Type if data is a file
 596         :return: None
 597         """
 598         accept = cherrypy.request.headers.get("Accept")
 599         if data is None:
 600             if accept and "text/html" in accept:
 601                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 602             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 603             return
 604         elif hasattr(data, "read"):  # file object
 605             if _format:
 606                 cherrypy.response.headers["Content-Type"] = _format
 607             elif "b" in data.mode:  # binariy asssumig zip
 608                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 609             else:
 610                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 611             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 612             return data
 613         if accept:
 614             if "application/json" in accept:
 615                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 616                 a = json.dumps(data, indent=4) + "\n"
 617                 return a.encode("utf8")
 618             elif "text/html" in accept:
 619                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 620
 621             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 622                 pass
 623             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 624             elif cherrypy.response.status >= 400:
 625                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 626                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 627                                          "for output format are available")
 628         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 629         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 630                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 631
 632     @cherrypy.expose
 633     def index(self, *args, **kwargs):
 634         token_info = None
 635         try:
 636             if cherrypy.request.method == "GET":
 637                 token_info = self.authenticator.authorize()
 638                 outdata = token_info   # Home page
 639             else:
 640                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 641                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 642
 643             return self._format_out(outdata, token_info)
 644
 645         except (EngineException, AuthException) as e:
 646             # cherrypy.log("index Exception {}".format(e))
 647             cherrypy.response.status = e.http_code.value
 648             return self._format_out("Welcome to OSM!", token_info)
 649
 650     @cherrypy.expose
 651     def version(self, *args, **kwargs):
 652         # TODO consider to remove and provide version using the static version file
 653         try:
 654             if cherrypy.request.method != "GET":
 655                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 656             elif args or kwargs:
 657                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 658             # TODO include version of other modules, pick up from some kafka admin message
 659             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 660             return self._format_out(osm_nbi_version)
 661         except NbiException as e:
 662             cherrypy.response.status = e.http_code.value
 663             problem_details = {
 664                 "code": e.http_code.name,
 665                 "status": e.http_code.value,
 666                 "detail": str(e),
 667             }
 668             return self._format_out(problem_details, None)
 669
 670     def domain(self):
 671         try:
 672             domains = {
 673                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 674                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 675             return self._format_out(domains)
 676         except NbiException as e:
 677             cherrypy.response.status = e.http_code.value
 678             problem_details = {
 679                 "code": e.http_code.name,
 680                 "status": e.http_code.value,
 681                 "detail": str(e),
 682             }
 683             return self._format_out(problem_details, None)
 684
 685     @staticmethod
 686     def _format_login(token_info):
 687         """
 688         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 689         log this information
 690         :param token_info: Dictionary with token content
 691         :return: None
 692         """
 693         cherrypy.request.login = token_info.get("username", "-")
 694         if token_info.get("project_name"):
 695             cherrypy.request.login += "/" + token_info["project_name"]
 696         if token_info.get("id"):
 697             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 698
 699     @cherrypy.expose
 700     def token(self, method, token_id=None, kwargs=None):
 701         token_info = None
 702         # self.engine.load_dbase(cherrypy.request.app.config)
 703         indata = self._format_in(kwargs)
 704         if not isinstance(indata, dict):
 705             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 706
 707         if method == "GET":
 708             token_info = self.authenticator.authorize()
 709             # for logging
 710             self._format_login(token_info)
 711             if token_id:
 712                 outdata = self.authenticator.get_token(token_info, token_id)
 713             else:
 714                 outdata = self.authenticator.get_token_list(token_info)
 715         elif method == "POST":
 716             try:
 717                 token_info = self.authenticator.authorize()
 718             except Exception:
 719                 token_info = None
 720             if kwargs:
 721                 indata.update(kwargs)
 722             # This is needed to log the user when authentication fails
 723             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 724             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 725             cherrypy.session['Authorization'] = outdata["_id"]
 726             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 727             # for logging
 728             self._format_login(token_info)
 729
 730             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 731             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 732         elif method == "DELETE":
 733             if not token_id and "id" in kwargs:
 734                 token_id = kwargs["id"]
 735             elif not token_id:
 736                 token_info = self.authenticator.authorize()
 737                 # for logging
 738                 self._format_login(token_info)
 739                 token_id = token_info["_id"]
 740             outdata = self.authenticator.del_token(token_id)
 741             token_info = None
 742             cherrypy.session['Authorization'] = "logout"
 743             # cherrypy.response.cookie["Authorization"] = token_id
 744             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 745         else:
 746             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 747         return self._format_out(outdata, token_info)
 748
 749     @cherrypy.expose
 750     def test(self, *args, **kwargs):
 751         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 752                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 753             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 754             return "test URL is disabled"
 755         thread_info = None
 756         if args and args[0] == "help":
 757             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 758                    "sleep/<time>\nmessage/topic\n</pre></html>"
 759
 760         elif args and args[0] == "init":
 761             try:
 762                 # self.engine.load_dbase(cherrypy.request.app.config)
 763                 self.engine.create_admin()
 764                 return "Done. User 'admin', password 'admin' created"
 765             except Exception:
 766                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 767                 return self._format_out("Database already initialized")
 768         elif args and args[0] == "file":
 769             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 770                                                   "text/plain", "attachment")
 771         elif args and args[0] == "file2":
 772             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 773             f = open(f_path, "r")
 774             cherrypy.response.headers["Content-type"] = "text/plain"
 775             return f
 776
 777         elif len(args) == 2 and args[0] == "db-clear":
 778             deleted_info = self.engine.db.del_list(args[1], kwargs)
 779             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 780         elif len(args) and args[0] == "fs-clear":
 781             if len(args) >= 2:
 782                 folders = (args[1],)
 783             else:
 784                 folders = self.engine.fs.dir_ls(".")
 785             for folder in folders:
 786                 self.engine.fs.file_delete(folder)
 787             return ",".join(folders) + " folders deleted\n"
 788         elif args and args[0] == "login":
 789             if not cherrypy.request.headers.get("Authorization"):
 790                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 791                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 792         elif args and args[0] == "login2":
 793             if not cherrypy.request.headers.get("Authorization"):
 794                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 795                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 796         elif args and args[0] == "sleep":
 797             sleep_time = 5
 798             try:
 799                 sleep_time = int(args[1])
 800             except Exception:
 801                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 802                 return self._format_out("Database already initialized")
 803             thread_info = cherrypy.thread_data
 804             print(thread_info)
 805             time.sleep(sleep_time)
 806             # thread_info
 807         elif len(args) >= 2 and args[0] == "message":
 808             main_topic = args[1]
 809             return_text = "<html><pre>{} ->\n".format(main_topic)
 810             try:
 811                 if cherrypy.request.method == 'POST':
 812                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 813                     for k, v in to_send.items():
 814                         self.engine.msg.write(main_topic, k, v)
 815                         return_text += "  {}: {}\n".format(k, v)
 816                 elif cherrypy.request.method == 'GET':
 817                     for k, v in kwargs.items():
 818                         v_dict = yaml.load(v, Loader=yaml.SafeLoader)
 819                         self.engine.msg.write(main_topic, k, v_dict)
 820                         return_text += "  {}: {}\n".format(k, v_dict)
 821             except Exception as e:
 822                 return_text += "Error: " + str(e)
 823             return_text += "</pre></html>\n"
 824             return return_text
 825
 826         return_text = (
 827             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 828             "  kwargs: {}\n".format(kwargs) +
 829             "  headers: {}\n".format(cherrypy.request.headers) +
 830             "  path_info: {}\n".format(cherrypy.request.path_info) +
 831             "  query_string: {}\n".format(cherrypy.request.query_string) +
 832             "  session: {}\n".format(cherrypy.session) +
 833             "  cookie: {}\n".format(cherrypy.request.cookie) +
 834             "  method: {}\n".format(cherrypy.request.method) +
 835             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 836             "  body:\n")
 837         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 838         if cherrypy.request.body.length:
 839             return_text += "    content: {}\n".format(
 840                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 841         if thread_info:
 842             return_text += "thread: {}\n".format(thread_info)
 843         return_text += "</pre></html>"
 844         return return_text
 845
 846     @staticmethod
 847     def _check_valid_url_method(method, *args):
 848         if len(args) < 3:
 849             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 850
 851         reference = valid_url_methods
 852         for arg in args:
 853             if arg is None:
 854                 break
 855             if not isinstance(reference, dict):
 856                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 857                                    HTTPStatus.METHOD_NOT_ALLOWED)
 858
 859             if arg in reference:
 860                 reference = reference[arg]
 861             elif "<ID>" in reference:
 862                 reference = reference["<ID>"]
 863             elif "*" in reference:
 864                 reference = reference["*"]
 865                 break
 866             else:
 867                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 868         if "TODO" in reference and method in reference["TODO"]:
 869             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 870         elif "METHODS" in reference and method not in reference["METHODS"]:
 871             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 872         return reference["ROLE_PERMISSION"] + method.lower()
 873
 874     @staticmethod
 875     def _set_location_header(main_topic, version, topic, id):
 876         """
 877         Insert response header Location with the URL of created item base on URL params
 878         :param main_topic:
 879         :param version:
 880         :param topic:
 881         :param id:
 882         :return: None
 883         """
 884         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 885         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 886         return
 887
 888     @staticmethod
 889     def _extract_query_string_operations(kwargs, method):
 890         """
 891
 892         :param kwargs:
 893         :return:
 894         """
 895         query_string_operations = []
 896         if kwargs:
 897             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 898                 if qs in kwargs and kwargs[qs].lower() != "false":
 899                     query_string_operations.append(qs.lower() + ":" + method.lower())
 900         return query_string_operations
 901
 902     @staticmethod
 903     def _manage_admin_query(token_info, kwargs, method, _id):
 904         """
 905         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 906         Check that users has rights to use them and returs the admin_query
 907         :param token_info: token_info rights obtained by token
 908         :param kwargs: query string input.
 909         :param method: http method: GET, POSST, PUT, ...
 910         :param _id:
 911         :return: admin_query dictionary with keys:
 912             public: True, False or None
 913             force: True or False
 914             project_id: tuple with projects used for accessing an element
 915             set_project: tuple with projects that a created element will belong to
 916             method: show, list, delete, write
 917         """
 918         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 919                        "admin": token_info["admin"], "public": None,
 920                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 921         if kwargs:
 922             # FORCE
 923             if "FORCE" in kwargs:
 924                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 925                     admin_query["force"] = True
 926                 del kwargs["FORCE"]
 927             # PUBLIC
 928             if "PUBLIC" in kwargs:
 929                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 930                     admin_query["public"] = True
 931                 else:
 932                     admin_query["public"] = False
 933                 del kwargs["PUBLIC"]
 934             # ADMIN
 935             if "ADMIN" in kwargs:
 936                 behave_as = kwargs.pop("ADMIN")
 937                 if behave_as.lower() != "false":
 938                     if not token_info["admin"]:
 939                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 940                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 941                         admin_query["project_id"] = ()
 942                     elif isinstance(behave_as, (list, tuple)):
 943                         admin_query["project_id"] = behave_as
 944                     else:   # isinstance(behave_as, str)
 945                         admin_query["project_id"] = (behave_as, )
 946             if "SET_PROJECT" in kwargs:
 947                 set_project = kwargs.pop("SET_PROJECT")
 948                 if not set_project:
 949                     admin_query["set_project"] = list(admin_query["project_id"])
 950                 else:
 951                     if isinstance(set_project, str):
 952                         set_project = (set_project, )
 953                     if admin_query["project_id"]:
 954                         for p in set_project:
 955                             if p not in admin_query["project_id"]:
 956                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 957                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 958                     admin_query["set_project"] = set_project
 959
 960             # PROJECT_READ
 961             # if "PROJECT_READ" in kwargs:
 962             #     admin_query["project"] = kwargs.pop("project")
 963             #     if admin_query["project"] == token_info["project_id"]:
 964         if method == "GET":
 965             if _id:
 966                 admin_query["method"] = "show"
 967             else:
 968                 admin_query["method"] = "list"
 969         elif method == "DELETE":
 970             admin_query["method"] = "delete"
 971         else:
 972             admin_query["method"] = "write"
 973         return admin_query
 974
 975     @cherrypy.expose
 976     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 977         token_info = None
 978         outdata = None
 979         _format = None
 980         method = "DONE"
 981         engine_topic = None
 982         rollback = []
 983         engine_session = None
 984         try:
 985             if not main_topic or not version or not topic:
 986                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 987                                    HTTPStatus.METHOD_NOT_ALLOWED)
 988             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 989                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 990                                    HTTPStatus.METHOD_NOT_ALLOWED)
 991             if version != 'v1':
 992                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 993
 994             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 995                 method = kwargs.pop("METHOD")
 996             else:
 997                 method = cherrypy.request.method
 998
 999             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
1000             query_string_operations = self._extract_query_string_operations(kwargs, method)
1001             if main_topic == "admin" and topic == "tokens":
1002                 return self.token(method, _id, kwargs)
1003             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
1004             if main_topic == "admin" and topic == "domains":
1005                 return self.domain()
1006             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
1007             indata = self._format_in(kwargs)
1008             engine_topic = topic
1009             if topic == "subscriptions":
1010                 engine_topic = main_topic + "_" + topic
1011             if item and topic != "pm_jobs":
1012                 engine_topic = item
1013
1014             if main_topic == "nsd":
1015                 engine_topic = "nsds"
1016             elif main_topic == "vnfpkgm":
1017                 engine_topic = "vnfds"
1018                 if topic == "vnfpkg_op_occs":
1019                     engine_topic = "vnfpkgops"
1020                 if topic == "vnf_packages" and item == "action":
1021                     engine_topic = "vnfpkgops"
1022             elif main_topic == "nslcm":
1023                 engine_topic = "nsrs"
1024                 if topic == "ns_lcm_op_occs":
1025                     engine_topic = "nslcmops"
1026                 if topic == "vnfrs" or topic == "vnf_instances":
1027                     engine_topic = "vnfrs"
1028             elif main_topic == "nst":
1029                 engine_topic = "nsts"
1030             elif main_topic == "nsilcm":
1031                 engine_topic = "nsis"
1032                 if topic == "nsi_lcm_op_occs":
1033                     engine_topic = "nsilcmops"
1034             elif main_topic == "pdu":
1035                 engine_topic = "pdus"
1036             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1037                 engine_topic = "vim_accounts"
1038
1039             if method == "GET":
1040                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1041                     if item in ("vnfd", "nsd", "nst"):
1042                         path = "$DESCRIPTOR"
1043                     elif args:
1044                         path = args
1045                     elif item == "artifacts":
1046                         path = ()
1047                     else:
1048                         path = None
1049                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1050                                                          cherrypy.request.headers.get("Accept"))
1051                     outdata = file
1052                 elif not _id:
1053                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1054                 else:
1055                     if item == "reports":
1056                         # TODO check that project_id (_id in this context) has permissions
1057                         _id = args[0]
1058                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1059
1060             elif method == "POST":
1061                 cherrypy.response.status = HTTPStatus.CREATED.value
1062                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1063                     _id = cherrypy.request.headers.get("Transaction-Id")
1064                     if not _id:
1065                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1066                                                       cherrypy.request.headers)
1067                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1068                                                            cherrypy.request.headers)
1069                     if completed:
1070                         self._set_location_header(main_topic, version, topic, _id)
1071                     else:
1072                         cherrypy.response.headers["Transaction-Id"] = _id
1073                     outdata = {"id": _id}
1074                 elif topic == "ns_instances_content":
1075                     # creates NSR
1076                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1077                     # creates nslcmop
1078                     indata["lcmOperationType"] = "instantiate"
1079                     indata["nsInstanceId"] = _id
1080                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1081                     self._set_location_header(main_topic, version, topic, _id)
1082                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1083                 elif topic == "ns_instances" and item:
1084                     indata["lcmOperationType"] = item
1085                     indata["nsInstanceId"] = _id
1086                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1087                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1088                     outdata = {"id": _id}
1089                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1090                 elif topic == "netslice_instances_content":
1091                     # creates NetSlice_Instance_record (NSIR)
1092                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1093                     self._set_location_header(main_topic, version, topic, _id)
1094                     indata["lcmOperationType"] = "instantiate"
1095                     indata["netsliceInstanceId"] = _id
1096                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1097                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1098                 elif topic == "netslice_instances" and item:
1099                     indata["lcmOperationType"] = item
1100                     indata["netsliceInstanceId"] = _id
1101                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1102                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1103                     outdata = {"id": _id}
1104                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1105                 elif topic == "vnf_packages" and item == "action":
1106                     indata["lcmOperationType"] = item
1107                     indata["vnfPkgId"] = _id
1108                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1109                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1110                     outdata = {"id": _id}
1111                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1112                 else:
1113                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1114                                                       cherrypy.request.headers)
1115                     self._set_location_header(main_topic, version, topic, _id)
1116                     outdata = {"id": _id}
1117                     if op_id:
1118                         outdata["op_id"] = op_id
1119                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1120                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1121
1122             elif method == "DELETE":
1123                 if not _id:
1124                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1125                     cherrypy.response.status = HTTPStatus.OK.value
1126                 else:  # len(args) > 1
1127                     # for NS NSI generate an operation
1128                     op_id = None
1129                     if topic == "ns_instances_content" and not engine_session["force"]:
1130                         nslcmop_desc = {
1131                             "lcmOperationType": "terminate",
1132                             "nsInstanceId": _id,
1133                             "autoremove": True
1134                         }
1135                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1136                         if op_id:
1137                             outdata = {"_id": op_id}
1138                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1139                         nsilcmop_desc = {
1140                             "lcmOperationType": "terminate",
1141                             "netsliceInstanceId": _id,
1142                             "autoremove": True
1143                         }
1144                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1145                         if op_id:
1146                             outdata = {"_id": op_id}
1147                     # if there is not any deletion in process, delete
1148                     if not op_id:
1149                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1150                         if op_id:
1151                             outdata = {"op_id": op_id}
1152                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1153
1154             elif method in ("PUT", "PATCH"):
1155                 op_id = None
1156                 if not indata and not kwargs and not engine_session.get("set_project"):
1157                     raise NbiException("Nothing to update. Provide payload and/or query string",
1158                                        HTTPStatus.BAD_REQUEST)
1159                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1160                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1161                                                            cherrypy.request.headers)
1162                     if not completed:
1163                         cherrypy.response.headers["Transaction-Id"] = id
1164                 else:
1165                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1166
1167                 if op_id:
1168                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1169                     outdata = {"op_id": op_id}
1170                 else:
1171                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1172                     outdata = None
1173             else:
1174                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1175
1176             # if Role information changes, it is needed to reload the information of roles
1177             if topic == "roles" and method != "GET":
1178                 self.authenticator.load_operation_to_allowed_roles()
1179
1180             if topic == "projects" and method == "DELETE" \
1181                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1182                 self.authenticator.remove_token_from_cache()
1183
1184             return self._format_out(outdata, token_info, _format)
1185         except Exception as e:
1186             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1187                               ValidationError, AuthconnException)):
1188                 http_code_value = cherrypy.response.status = e.http_code.value
1189                 http_code_name = e.http_code.name
1190                 cherrypy.log("Exception {}".format(e))
1191             else:
1192                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1193                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1194                 http_code_name = HTTPStatus.BAD_REQUEST.name
1195             if hasattr(outdata, "close"):  # is an open file
1196                 outdata.close()
1197             error_text = str(e)
1198             rollback.reverse()
1199             for rollback_item in rollback:
1200                 try:
1201                     if rollback_item.get("operation") == "set":
1202                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1203                                                rollback_item["content"], fail_on_empty=False)
1204                     else:
1205                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1206                                                fail_on_empty=False)
1207                 except Exception as e2:
1208                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1209                     cherrypy.log(rollback_error_text)
1210                     error_text += ". " + rollback_error_text
1211             # if isinstance(e, MsgException):
1212             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1213             #         engine_topic[:-1], method, error_text)
1214             problem_details = {
1215                 "code": http_code_name,
1216                 "status": http_code_value,
1217                 "detail": error_text,
1218             }
1219             return self._format_out(problem_details, token_info)
1220             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1221         finally:
1222             if token_info:
1223                 self._format_login(token_info)
1224                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1225                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1226                         if outdata.get(logging_id):
1227                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1228
1229
1230 def _start_service():
1231     """
1232     Callback function called when cherrypy.engine starts
1233     Override configuration with env variables
1234     Set database, storage, message configuration
1235     Init database with admin/admin user password
1236     """
1237     global nbi_server
1238     global subscription_thread
1239     cherrypy.log.error("Starting osm_nbi")
1240     # update general cherrypy configuration
1241     update_dict = {}
1242
1243     engine_config = cherrypy.tree.apps['/osm'].config
1244     for k, v in environ.items():
1245         if not k.startswith("OSMNBI_"):
1246             continue
1247         k1, _, k2 = k[7:].lower().partition("_")
1248         if not k2:
1249             continue
1250         try:
1251             # update static configuration
1252             if k == 'OSMNBI_STATIC_DIR':
1253                 engine_config["/static"]['tools.staticdir.dir'] = v
1254                 engine_config["/static"]['tools.staticdir.on'] = True
1255             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1256                 update_dict['server.socket_port'] = int(v)
1257             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1258                 update_dict['server.socket_host'] = v
1259             elif k1 in ("server", "test", "auth", "log"):
1260                 update_dict[k1 + '.' + k2] = v
1261             elif k1 in ("message", "database", "storage", "authentication"):
1262                 # k2 = k2.replace('_', '.')
1263                 if k2 in ("port", "db_port"):
1264                     engine_config[k1][k2] = int(v)
1265                 else:
1266                     engine_config[k1][k2] = v
1267
1268         except ValueError as e:
1269             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1270         except Exception as e:
1271             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1272
1273     if update_dict:
1274         cherrypy.config.update(update_dict)
1275         engine_config["global"].update(update_dict)
1276
1277     # logging cherrypy
1278     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1279     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1280     logger_server = logging.getLogger("cherrypy.error")
1281     logger_access = logging.getLogger("cherrypy.access")
1282     logger_cherry = logging.getLogger("cherrypy")
1283     logger_nbi = logging.getLogger("nbi")
1284
1285     if "log.file" in engine_config["global"]:
1286         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1287                                                             maxBytes=100e6, backupCount=9, delay=0)
1288         file_handler.setFormatter(log_formatter_simple)
1289         logger_cherry.addHandler(file_handler)
1290         logger_nbi.addHandler(file_handler)
1291     # log always to standard output
1292     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1293                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1294                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1295                             }.items():
1296         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1297         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1298         str_handler = logging.StreamHandler()
1299         str_handler.setFormatter(log_formatter_cherry)
1300         logger.addHandler(str_handler)
1301
1302     if engine_config["global"].get("log.level"):
1303         logger_cherry.setLevel(engine_config["global"]["log.level"])
1304         logger_nbi.setLevel(engine_config["global"]["log.level"])
1305
1306     # logging other modules
1307     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1308         engine_config[k1]["logger_name"] = logname
1309         logger_module = logging.getLogger(logname)
1310         if "logfile" in engine_config[k1]:
1311             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1312                                                                 maxBytes=100e6, backupCount=9, delay=0)
1313             file_handler.setFormatter(log_formatter_simple)
1314             logger_module.addHandler(file_handler)
1315         if "loglevel" in engine_config[k1]:
1316             logger_module.setLevel(engine_config[k1]["loglevel"])
1317     # TODO add more entries, e.g.: storage
1318     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1319     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1320     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1321     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1322
1323     # start subscriptions thread:
1324     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1325     subscription_thread.start()
1326     # Do not capture except SubscriptionException
1327
1328     backend = engine_config["authentication"]["backend"]
1329     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1330                        .format(nbi_version, nbi_version_date, backend))
1331
1332
1333 def _stop_service():
1334     """
1335     Callback function called when cherrypy.engine stops
1336     TODO: Ending database connections.
1337     """
1338     global subscription_thread
1339     if subscription_thread:
1340         subscription_thread.terminate()
1341     subscription_thread = None
1342     cherrypy.tree.apps['/osm'].root.engine.stop()
1343     cherrypy.log.error("Stopping osm_nbi")
1344
1345
1346 def nbi(config_file):
1347     global nbi_server
1348     # conf = {
1349     #     '/': {
1350     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1351     #         'tools.sessions.on': True,
1352     #         'tools.response_headers.on': True,
1353     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1354     #     }
1355     # }
1356     # cherrypy.Server.ssl_module = 'builtin'
1357     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1358     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1359     # cherrypy.Server.thread_pool = 10
1360     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1361
1362     # cherrypy.config.update({'tools.auth_basic.on': True,
1363     #    'tools.auth_basic.realm': 'localhost',
1364     #    'tools.auth_basic.checkpassword': validate_password})
1365     nbi_server = Server()
1366     cherrypy.engine.subscribe('start', _start_service)
1367     cherrypy.engine.subscribe('stop', _stop_service)
1368     cherrypy.quickstart(nbi_server, '/osm', config_file)
1369
1370
1371 def usage():
1372     print("""Usage: {} [options]
1373         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1374         -h|--help: shows this help
1375         """.format(sys.argv[0]))
1376     # --log-socket-host HOST: send logs to this host")
1377     # --log-socket-port PORT: send logs using this port (default: 9022)")
1378
1379
1380 if __name__ == '__main__':
1381     try:
1382         # load parameters and configuration
1383         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1384         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1385         config_file = None
1386         for o, a in opts:
1387             if o in ("-h", "--help"):
1388                 usage()
1389                 sys.exit()
1390             elif o in ("-c", "--config"):
1391                 config_file = a
1392             # elif o == "--log-socket-port":
1393             #     log_socket_port = a
1394             # elif o == "--log-socket-host":
1395             #     log_socket_host = a
1396             # elif o == "--log-file":
1397             #     log_file = a
1398             else:
1399                 assert False, "Unhandled option"
1400         if config_file:
1401             if not path.isfile(config_file):
1402                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1403                 exit(1)
1404         else:
1405             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1406                 if path.isfile(config_file):
1407                     break
1408             else:
1409                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1410                 exit(1)
1411         nbi(config_file)
1412     except getopt.GetoptError as e:
1413         print(str(e), file=sys.stderr)
1414         # usage()
1415         exit(1)