osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50 """
  51 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  52 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  53         /nsd/v1
  54             /ns_descriptors_content                             O       O
  55                 /<nsdInfoId>                                    O       O       O       O
  56             /ns_descriptors                                     O5      O5
  57                 /<nsdInfoId>                                    O5                      O5      5
  58                     /nsd_content                                O5              O5
  59                     /nsd                                        O
  60                     /artifacts[/<artifactPath>]                 O
  61             /pnf_descriptors                                    5       5
  62                 /<pnfdInfoId>                                   5                       5       5
  63                     /pnfd_content                               5               5
  64             /subscriptions                                      5       5
  65                 /<subscriptionId>                               5                       X
  66
  67         /vnfpkgm/v1
  68             /vnf_packages_content                               O       O
  69                 /<vnfPkgId>                                     O                       O
  70             /vnf_packages                                       O5      O5
  71                 /<vnfPkgId>                                     O5                      O5      5
  72                     /package_content                            O5               O5
  73                         /upload_from_uri                                X
  74                     /vnfd                                       O5
  75                     /artifacts[/<artifactPath>]                 O5
  76             /subscriptions                                      X       X
  77                 /<subscriptionId>                               X                       X
  78
  79         /nslcm/v1
  80             /ns_instances_content                               O       O
  81                 /<nsInstanceId>                                 O                       O
  82             /ns_instances                                       5       5
  83                 /<nsInstanceId>                                 O5                      O5
  84                     instantiate                                         O5
  85                     terminate                                           O5
  86                     action                                              O
  87                     scale                                               O5
  88                     heal                                                5
  89             /ns_lcm_op_occs                                     5       5
  90                 /<nsLcmOpOccId>                                 5                       5       5
  91                     TO BE COMPLETED                             5               5
  92             /vnf_instances  (also vnfrs for compatibility)      O
  93                 /<vnfInstanceId>                                O
  94             /subscriptions                                      5       5
  95                 /<subscriptionId>                               5                       X
  96
  97         /pdu/v1
  98             /pdu_descriptors                                    O       O
  99                 /<id>                                           O               O       O       O
 100
 101         /admin/v1
 102             /tokens                                             O       O
 103                 /<id>                                           O                       O
 104             /users                                              O       O
 105                 /<id>                                           O               O       O       O
 106             /projects                                           O       O
 107                 /<id>                                           O                       O
 108             /vim_accounts  (also vims for compatibility)        O       O
 109                 /<id>                                           O                       O       O
 110             /wim_accounts                                       O       O
 111                 /<id>                                           O                       O       O
 112             /sdns                                               O       O
 113                 /<id>                                           O                       O       O
 114             /k8sclusters                                        O       O
 115                 /<id>                                           O                       O       O
 116             /k8srepos                                           O       O
 117                 /<id>                                           O                               O
 118             /osmrepos                                           O       O
 119                 /<id>                                           O                               O
 120
 121         /nst/v1                                                 O       O
 122             /netslice_templates_content                         O       O
 123                 /<nstInfoId>                                    O       O       O       O
 124             /netslice_templates                                 O       O
 125                 /<nstInfoId>                                    O                       O       O
 126                     /nst_content                                O               O
 127                     /nst                                        O
 128                     /artifacts[/<artifactPath>]                 O
 129             /subscriptions                                      X       X
 130                 /<subscriptionId>                               X                       X
 131
 132         /nsilcm/v1
 133             /netslice_instances_content                         O       O
 134                 /<SliceInstanceId>                              O                       O
 135             /netslice_instances                                 O       O
 136                 /<SliceInstanceId>                              O                       O
 137                     instantiate                                         O
 138                     terminate                                           O
 139                     action                                              O
 140             /nsi_lcm_op_occs                                    O       O
 141                 /<nsiLcmOpOccId>                                O                       O       O
 142             /subscriptions                                      X       X
 143                 /<subscriptionId>                               X                       X
 144
 145 query string:
 146     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 147         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 148         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 149         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 150         attrName := string
 151     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 152     item of the array, that is, pass if any item of the array pass the filter.
 153     It allows both ne and neq for not equal
 154     TODO: 4.3.3 Attribute selectors
 155         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 156         (none)  … same as “exclude_default”
 157         all_fields      … all attributes.
 158         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 159         conditionally mandatory, and that are not provided in <list>.
 160         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 161         are not conditionally mandatory, and that are provided in <list>.
 162         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 163         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 164         the particular resource
 165         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 166         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 167         present specification for the particular resource, but that are not part of <list>
 168     Additionally it admits some administrator values:
 169         FORCE: To force operations skipping dependency checkings
 170         ADMIN: To act as an administrator or a different project
 171         PUBLIC: To get public descriptors or set a descriptor as public
 172         SET_PROJECT: To make a descriptor available for other project
 173
 174 Header field name       Reference       Example Descriptions
 175     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 176     This header field shall be present if the response is expected to have a non-empty message body.
 177     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 178     This header field shall be present if the request has a non-empty message body.
 179     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 180     Details are specified in clause 4.5.3.
 181     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 182 Header field name       Reference       Example Descriptions
 183     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 184     This header field shall be present if the response has a non-empty message body.
 185     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 186     new resource has been created.
 187     This header field shall be present if the response status code is 201 or 3xx.
 188     In the present document this header field is also used if the response status code is 202 and a new resource was
 189     created.
 190     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 191     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 192     token.
 193     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 194     certain resources.
 195     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 196     response, and the total length of the file.
 197     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 198 """
 199
 200 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 201 # ^ Contains possible administrative query string words:
 202 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 203 #           (not owned by my session project).
 204 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 205 #     FORCE=True(by default)|False: Force edition/deletion operations
 206 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 207
 208 valid_url_methods = {
 209     # contains allowed URL and methods, and the role_permission name
 210     "admin": {
 211         "v1": {
 212             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 213                        "ROLE_PERMISSION": "tokens:",
 214                        "<ID>": {"METHODS": ("GET", "DELETE"),
 215                                 "ROLE_PERMISSION": "tokens:id:"
 216                                 }
 217                        },
 218             "users": {"METHODS": ("GET", "POST"),
 219                       "ROLE_PERMISSION": "users:",
 220                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 221                                "ROLE_PERMISSION": "users:id:"
 222                                }
 223                       },
 224             "projects": {"METHODS": ("GET", "POST"),
 225                          "ROLE_PERMISSION": "projects:",
 226                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 227                                   "ROLE_PERMISSION": "projects:id:"}
 228                          },
 229             "roles": {"METHODS": ("GET", "POST"),
 230                       "ROLE_PERMISSION": "roles:",
 231                       "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 232                                "ROLE_PERMISSION": "roles:id:"
 233                                }
 234                       },
 235             "vims": {"METHODS": ("GET", "POST"),
 236                      "ROLE_PERMISSION": "vims:",
 237                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 238                               "ROLE_PERMISSION": "vims:id:"
 239                               }
 240                      },
 241             "vim_accounts": {"METHODS": ("GET", "POST"),
 242                              "ROLE_PERMISSION": "vim_accounts:",
 243                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 244                                       "ROLE_PERMISSION": "vim_accounts:id:"
 245                                       }
 246                              },
 247             "wim_accounts": {"METHODS": ("GET", "POST"),
 248                              "ROLE_PERMISSION": "wim_accounts:",
 249                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 250                                       "ROLE_PERMISSION": "wim_accounts:id:"
 251                                       }
 252                              },
 253             "sdns": {"METHODS": ("GET", "POST"),
 254                      "ROLE_PERMISSION": "sdn_controllers:",
 255                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 256                               "ROLE_PERMISSION": "sdn_controllers:id:"
 257                               }
 258                      },
 259             "k8sclusters": {"METHODS": ("GET", "POST"),
 260                             "ROLE_PERMISSION": "k8sclusters:",
 261                             "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 262                                      "ROLE_PERMISSION": "k8sclusters:id:"
 263                                      }
 264                             },
 265             "k8srepos": {"METHODS": ("GET", "POST"),
 266                          "ROLE_PERMISSION": "k8srepos:",
 267                          "<ID>": {"METHODS": ("GET", "DELETE"),
 268                                   "ROLE_PERMISSION": "k8srepos:id:"
 269                                   }
 270                          },
 271             "osmrepos": {"METHODS": ("GET", "POST"),
 272                          "ROLE_PERMISSION": "osmrepos:",
 273                          "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 274                                   "ROLE_PERMISSION": "osmrepos:id:"
 275                                   }
 276                          },
 277             "domains": {"METHODS": ("GET", ),
 278                         "ROLE_PERMISSION": "domains:",
 279                         },
 280         }
 281     },
 282     "pdu": {
 283         "v1": {
 284             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 285                                 "ROLE_PERMISSION": "pduds:",
 286                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 287                                          "ROLE_PERMISSION": "pduds:id:"
 288                                          }
 289                                 },
 290         }
 291     },
 292     "nsd": {
 293         "v1": {
 294             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 295                                        "ROLE_PERMISSION": "nsds:",
 296                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 297                                                 "ROLE_PERMISSION": "nsds:id:"
 298                                                 }
 299                                        },
 300             "ns_descriptors": {"METHODS": ("GET", "POST"),
 301                                "ROLE_PERMISSION": "nsds:",
 302                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 303                                         "ROLE_PERMISSION": "nsds:id:",
 304                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 305                                                         "ROLE_PERMISSION": "nsds:id:content:",
 306                                                         },
 307                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 308                                                 "ROLE_PERMISSION": "nsds:id:content:"
 309                                                 },
 310                                         "artifacts": {"*": {"METHODS": ("GET",),
 311                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 312                                                             }
 313                                                       }
 314                                         }
 315                                },
 316             "pnf_descriptors": {"TODO": ("GET", "POST"),
 317                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 318                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 319                                          }
 320                                 },
 321             "subscriptions": {"TODO": ("GET", "POST"),
 322                               "<ID>": {"TODO": ("GET", "DELETE")}
 323                               },
 324         }
 325     },
 326     "vnfpkgm": {
 327         "v1": {
 328             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 329                                      "ROLE_PERMISSION": "vnfds:",
 330                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 331                                               "ROLE_PERMISSION": "vnfds:id:"}
 332                                      },
 333             "vnf_packages": {"METHODS": ("GET", "POST"),
 334                              "ROLE_PERMISSION": "vnfds:",
 335                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 336                                       "ROLE_PERMISSION": "vnfds:id:",
 337                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 338                                                           "ROLE_PERMISSION": "vnfds:id:",
 339                                                           "upload_from_uri": {"METHODS": (),
 340                                                                               "TODO": ("POST", ),
 341                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 342                                                                               }
 343                                                           },
 344                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 345                                                "ROLE_PERMISSION": "vnfds:id:content:"
 346                                                },
 347                                       "artifacts": {"*": {"METHODS": ("GET", ),
 348                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 349                                                           }
 350                                                     },
 351                                       "action": {"METHODS": ("POST", ),
 352                                                  "ROLE_PERMISSION": "vnfds:id:action:"
 353                                                  },
 354                                       }
 355                              },
 356             "subscriptions": {"TODO": ("GET", "POST"),
 357                               "<ID>": {"TODO": ("GET", "DELETE")}
 358                               },
 359             "vnfpkg_op_occs": {"METHODS": ("GET", ),
 360                                "ROLE_PERMISSION": "vnfds:vnfpkgops:",
 361                                "<ID>": {"METHODS": ("GET", ),
 362                                         "ROLE_PERMISSION": "vnfds:vnfpkgops:id:"
 363                                         }
 364                                },
 365         }
 366     },
 367     "nslcm": {
 368         "v1": {
 369             "ns_instances_content": {"METHODS": ("GET", "POST"),
 370                                      "ROLE_PERMISSION": "ns_instances:",
 371                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 372                                               "ROLE_PERMISSION": "ns_instances:id:"
 373                                               }
 374                                      },
 375             "ns_instances": {"METHODS": ("GET", "POST"),
 376                              "ROLE_PERMISSION": "ns_instances:",
 377                              "<ID>": {"METHODS": ("GET", "DELETE"),
 378                                       "ROLE_PERMISSION": "ns_instances:id:",
 379                                       "scale": {"METHODS": ("POST",),
 380                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 381                                                 },
 382                                       "terminate": {"METHODS": ("POST",),
 383                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 384                                                     },
 385                                       "instantiate": {"METHODS": ("POST",),
 386                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 387                                                       },
 388                                       "action": {"METHODS": ("POST",),
 389                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 390                                                  },
 391                                       }
 392                              },
 393             "ns_lcm_op_occs": {"METHODS": ("GET",),
 394                                "ROLE_PERMISSION": "ns_instances:opps:",
 395                                "<ID>": {"METHODS": ("GET",),
 396                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 397                                         },
 398                                },
 399             "vnfrs": {"METHODS": ("GET",),
 400                       "ROLE_PERMISSION": "vnf_instances:",
 401                       "<ID>": {"METHODS": ("GET",),
 402                                "ROLE_PERMISSION": "vnf_instances:id:"
 403                                }
 404                       },
 405             "vnf_instances": {"METHODS": ("GET",),
 406                               "ROLE_PERMISSION": "vnf_instances:",
 407                               "<ID>": {"METHODS": ("GET",),
 408                                        "ROLE_PERMISSION": "vnf_instances:id:"
 409                                        }
 410                               },
 411             "subscriptions": {"METHODS": ("GET", "POST"),
 412                               "ROLE_PERMISSION": "ns_subscriptions:",
 413                               "<ID>": {"METHODS": ("GET", "DELETE"),
 414                                        "ROLE_PERMISSION": "ns_subscriptions:id:"
 415                                        }
 416                               },
 417         }
 418     },
 419     "nst": {
 420         "v1": {
 421             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 422                                            "ROLE_PERMISSION": "slice_templates:",
 423                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 424                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 425                                            },
 426             "netslice_templates": {"METHODS": ("GET", "POST"),
 427                                    "ROLE_PERMISSION": "slice_templates:",
 428                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 429                                             "TODO": ("PATCH",),
 430                                             "ROLE_PERMISSION": "slice_templates:id:",
 431                                             "nst_content": {"METHODS": ("GET", "PUT"),
 432                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 433                                                             },
 434                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 435                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 436                                                     },
 437                                             "artifacts": {"*": {"METHODS": ("GET",),
 438                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 439                                                                 }
 440                                                           }
 441                                             }
 442                                    },
 443             "subscriptions": {"TODO": ("GET", "POST"),
 444                               "<ID>": {"TODO": ("GET", "DELETE")}
 445                               },
 446         }
 447     },
 448     "nsilcm": {
 449         "v1": {
 450             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 451                                            "ROLE_PERMISSION": "slice_instances:",
 452                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 453                                                     "ROLE_PERMISSION": "slice_instances:id:"
 454                                                     }
 455                                            },
 456             "netslice_instances": {"METHODS": ("GET", "POST"),
 457                                    "ROLE_PERMISSION": "slice_instances:",
 458                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 459                                             "ROLE_PERMISSION": "slice_instances:id:",
 460                                             "terminate": {"METHODS": ("POST",),
 461                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 462                                                           },
 463                                             "instantiate": {"METHODS": ("POST",),
 464                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 465                                                             },
 466                                             "action": {"METHODS": ("POST",),
 467                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 468                                                        },
 469                                             }
 470                                    },
 471             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 472                                 "ROLE_PERMISSION": "slice_instances:opps:",
 473                                 "<ID>": {"METHODS": ("GET",),
 474                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 475                                          },
 476                                 },
 477         }
 478     },
 479     "nspm": {
 480         "v1": {
 481             "pm_jobs": {
 482                 "<ID>": {
 483                     "reports": {
 484                         "<ID>": {"METHODS": ("GET",),
 485                                  "ROLE_PERMISSION": "reports:id:",
 486                                  }
 487                     }
 488                 },
 489             },
 490         },
 491     },
 492 }
 493
 494
 495 class NbiException(Exception):
 496
 497     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 498         Exception.__init__(self, message)
 499         self.http_code = http_code
 500
 501
 502 class Server(object):
 503     instance = 0
 504     # to decode bytes to str
 505     reader = getreader("utf-8")
 506
 507     def __init__(self):
 508         self.instance += 1
 509         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 510         self.engine = Engine(self.authenticator)
 511
 512     def _format_in(self, kwargs):
 513         try:
 514             indata = None
 515             if cherrypy.request.body.length:
 516                 error_text = "Invalid input format "
 517
 518                 if "Content-Type" in cherrypy.request.headers:
 519                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 520                         error_text = "Invalid json format "
 521                         indata = json.load(self.reader(cherrypy.request.body))
 522                         cherrypy.request.headers.pop("Content-File-MD5", None)
 523                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 524                         error_text = "Invalid yaml format "
 525                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 526                         cherrypy.request.headers.pop("Content-File-MD5", None)
 527                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 528                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 529                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 530                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 531                         indata = cherrypy.request.body  # .read()
 532                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 533                         if "descriptor_file" in kwargs:
 534                             filecontent = kwargs.pop("descriptor_file")
 535                             if not filecontent.file:
 536                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 537                             indata = filecontent.file  # .read()
 538                             if filecontent.content_type.value:
 539                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 540                     else:
 541                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 542                         #                          "Only 'Content-Type' of type 'application/json' or
 543                         # 'application/yaml' for input format are available")
 544                         error_text = "Invalid yaml format "
 545                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 546                         cherrypy.request.headers.pop("Content-File-MD5", None)
 547                 else:
 548                     error_text = "Invalid yaml format "
 549                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 550                     cherrypy.request.headers.pop("Content-File-MD5", None)
 551             if not indata:
 552                 indata = {}
 553
 554             format_yaml = False
 555             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 556                 format_yaml = True
 557
 558             for k, v in kwargs.items():
 559                 if isinstance(v, str):
 560                     if v == "":
 561                         kwargs[k] = None
 562                     elif format_yaml:
 563                         try:
 564                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 565                         except Exception:
 566                             pass
 567                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 568                         try:
 569                             kwargs[k] = int(v)
 570                         except Exception:
 571                             try:
 572                                 kwargs[k] = float(v)
 573                             except Exception:
 574                                 pass
 575                     elif v.find(",") > 0:
 576                         kwargs[k] = v.split(",")
 577                 elif isinstance(v, (list, tuple)):
 578                     for index in range(0, len(v)):
 579                         if v[index] == "":
 580                             v[index] = None
 581                         elif format_yaml:
 582                             try:
 583                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 584                             except Exception:
 585                                 pass
 586
 587             return indata
 588         except (ValueError, yaml.YAMLError) as exc:
 589             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 590         except KeyError as exc:
 591             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 592         except Exception as exc:
 593             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 594
 595     @staticmethod
 596     def _format_out(data, token_info=None, _format=None):
 597         """
 598         return string of dictionary data according to requested json, yaml, xml. By default json
 599         :param data: response to be sent. Can be a dict, text or file
 600         :param token_info: Contains among other username and project
 601         :param _format: The format to be set as Content-Type if data is a file
 602         :return: None
 603         """
 604         accept = cherrypy.request.headers.get("Accept")
 605         if data is None:
 606             if accept and "text/html" in accept:
 607                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 608             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 609             return
 610         elif hasattr(data, "read"):  # file object
 611             if _format:
 612                 cherrypy.response.headers["Content-Type"] = _format
 613             elif "b" in data.mode:  # binariy asssumig zip
 614                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 615             else:
 616                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 617             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 618             return data
 619         if accept:
 620             if "application/json" in accept:
 621                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 622                 a = json.dumps(data, indent=4) + "\n"
 623                 return a.encode("utf8")
 624             elif "text/html" in accept:
 625                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 626
 627             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 628                 pass
 629             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 630             elif cherrypy.response.status >= 400:
 631                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 632                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 633                                          "for output format are available")
 634         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 635         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 636                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 637
 638     @cherrypy.expose
 639     def index(self, *args, **kwargs):
 640         token_info = None
 641         try:
 642             if cherrypy.request.method == "GET":
 643                 token_info = self.authenticator.authorize()
 644                 outdata = token_info   # Home page
 645             else:
 646                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 647                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 648
 649             return self._format_out(outdata, token_info)
 650
 651         except (EngineException, AuthException) as e:
 652             # cherrypy.log("index Exception {}".format(e))
 653             cherrypy.response.status = e.http_code.value
 654             return self._format_out("Welcome to OSM!", token_info)
 655
 656     @cherrypy.expose
 657     def version(self, *args, **kwargs):
 658         # TODO consider to remove and provide version using the static version file
 659         try:
 660             if cherrypy.request.method != "GET":
 661                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 662             elif args or kwargs:
 663                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 664             # TODO include version of other modules, pick up from some kafka admin message
 665             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 666             return self._format_out(osm_nbi_version)
 667         except NbiException as e:
 668             cherrypy.response.status = e.http_code.value
 669             problem_details = {
 670                 "code": e.http_code.name,
 671                 "status": e.http_code.value,
 672                 "detail": str(e),
 673             }
 674             return self._format_out(problem_details, None)
 675
 676     def domain(self):
 677         try:
 678             domains = {
 679                 "user_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("user_domain_name"),
 680                 "project_domain_name": cherrypy.tree.apps['/osm'].config["authentication"].get("project_domain_name")}
 681             return self._format_out(domains)
 682         except NbiException as e:
 683             cherrypy.response.status = e.http_code.value
 684             problem_details = {
 685                 "code": e.http_code.name,
 686                 "status": e.http_code.value,
 687                 "detail": str(e),
 688             }
 689             return self._format_out(problem_details, None)
 690
 691     @staticmethod
 692     def _format_login(token_info):
 693         """
 694         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 695         log this information
 696         :param token_info: Dictionary with token content
 697         :return: None
 698         """
 699         cherrypy.request.login = token_info.get("username", "-")
 700         if token_info.get("project_name"):
 701             cherrypy.request.login += "/" + token_info["project_name"]
 702         if token_info.get("id"):
 703             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 704
 705     @cherrypy.expose
 706     def token(self, method, token_id=None, kwargs=None):
 707         token_info = None
 708         # self.engine.load_dbase(cherrypy.request.app.config)
 709         indata = self._format_in(kwargs)
 710         if not isinstance(indata, dict):
 711             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 712
 713         if method == "GET":
 714             token_info = self.authenticator.authorize()
 715             # for logging
 716             self._format_login(token_info)
 717             if token_id:
 718                 outdata = self.authenticator.get_token(token_info, token_id)
 719             else:
 720                 outdata = self.authenticator.get_token_list(token_info)
 721         elif method == "POST":
 722             try:
 723                 token_info = self.authenticator.authorize()
 724             except Exception:
 725                 token_info = None
 726             if kwargs:
 727                 indata.update(kwargs)
 728             # This is needed to log the user when authentication fails
 729             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 730             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 731             cherrypy.session['Authorization'] = outdata["_id"]
 732             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 733             # for logging
 734             self._format_login(token_info)
 735
 736             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 737             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 738         elif method == "DELETE":
 739             if not token_id and "id" in kwargs:
 740                 token_id = kwargs["id"]
 741             elif not token_id:
 742                 token_info = self.authenticator.authorize()
 743                 # for logging
 744                 self._format_login(token_info)
 745                 token_id = token_info["_id"]
 746             outdata = self.authenticator.del_token(token_id)
 747             token_info = None
 748             cherrypy.session['Authorization'] = "logout"
 749             # cherrypy.response.cookie["Authorization"] = token_id
 750             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 751         else:
 752             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 753         return self._format_out(outdata, token_info)
 754
 755     @cherrypy.expose
 756     def test(self, *args, **kwargs):
 757         if not cherrypy.config.get("server.enable_test") or (isinstance(cherrypy.config["server.enable_test"], str) and
 758                                                              cherrypy.config["server.enable_test"].lower() == "false"):
 759             cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
 760             return "test URL is disabled"
 761         thread_info = None
 762         if args and args[0] == "help":
 763             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 764                    "sleep/<time>\nmessage/topic\n</pre></html>"
 765
 766         elif args and args[0] == "init":
 767             try:
 768                 # self.engine.load_dbase(cherrypy.request.app.config)
 769                 self.engine.create_admin()
 770                 return "Done. User 'admin', password 'admin' created"
 771             except Exception:
 772                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 773                 return self._format_out("Database already initialized")
 774         elif args and args[0] == "file":
 775             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 776                                                   "text/plain", "attachment")
 777         elif args and args[0] == "file2":
 778             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 779             f = open(f_path, "r")
 780             cherrypy.response.headers["Content-type"] = "text/plain"
 781             return f
 782
 783         elif len(args) == 2 and args[0] == "db-clear":
 784             deleted_info = self.engine.db.del_list(args[1], kwargs)
 785             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 786         elif len(args) and args[0] == "fs-clear":
 787             if len(args) >= 2:
 788                 folders = (args[1],)
 789             else:
 790                 folders = self.engine.fs.dir_ls(".")
 791             for folder in folders:
 792                 self.engine.fs.file_delete(folder)
 793             return ",".join(folders) + " folders deleted\n"
 794         elif args and args[0] == "login":
 795             if not cherrypy.request.headers.get("Authorization"):
 796                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 797                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 798         elif args and args[0] == "login2":
 799             if not cherrypy.request.headers.get("Authorization"):
 800                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 801                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 802         elif args and args[0] == "sleep":
 803             sleep_time = 5
 804             try:
 805                 sleep_time = int(args[1])
 806             except Exception:
 807                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 808                 return self._format_out("Database already initialized")
 809             thread_info = cherrypy.thread_data
 810             print(thread_info)
 811             time.sleep(sleep_time)
 812             # thread_info
 813         elif len(args) >= 2 and args[0] == "message":
 814             main_topic = args[1]
 815             return_text = "<html><pre>{} ->\n".format(main_topic)
 816             try:
 817                 if cherrypy.request.method == 'POST':
 818                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 819                     for k, v in to_send.items():
 820                         self.engine.msg.write(main_topic, k, v)
 821                         return_text += "  {}: {}\n".format(k, v)
 822                 elif cherrypy.request.method == 'GET':
 823                     for k, v in kwargs.items():
 824                         v_dict = yaml.load(v, Loader=yaml.SafeLoader)
 825                         self.engine.msg.write(main_topic, k, v_dict)
 826                         return_text += "  {}: {}\n".format(k, v_dict)
 827             except Exception as e:
 828                 return_text += "Error: " + str(e)
 829             return_text += "</pre></html>\n"
 830             return return_text
 831
 832         return_text = (
 833             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 834             "  kwargs: {}\n".format(kwargs) +
 835             "  headers: {}\n".format(cherrypy.request.headers) +
 836             "  path_info: {}\n".format(cherrypy.request.path_info) +
 837             "  query_string: {}\n".format(cherrypy.request.query_string) +
 838             "  session: {}\n".format(cherrypy.session) +
 839             "  cookie: {}\n".format(cherrypy.request.cookie) +
 840             "  method: {}\n".format(cherrypy.request.method) +
 841             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 842             "  body:\n")
 843         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 844         if cherrypy.request.body.length:
 845             return_text += "    content: {}\n".format(
 846                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 847         if thread_info:
 848             return_text += "thread: {}\n".format(thread_info)
 849         return_text += "</pre></html>"
 850         return return_text
 851
 852     @staticmethod
 853     def _check_valid_url_method(method, *args):
 854         if len(args) < 3:
 855             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 856
 857         reference = valid_url_methods
 858         for arg in args:
 859             if arg is None:
 860                 break
 861             if not isinstance(reference, dict):
 862                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 863                                    HTTPStatus.METHOD_NOT_ALLOWED)
 864
 865             if arg in reference:
 866                 reference = reference[arg]
 867             elif "<ID>" in reference:
 868                 reference = reference["<ID>"]
 869             elif "*" in reference:
 870                 reference = reference["*"]
 871                 break
 872             else:
 873                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 874         if "TODO" in reference and method in reference["TODO"]:
 875             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 876         elif "METHODS" in reference and method not in reference["METHODS"]:
 877             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 878         return reference["ROLE_PERMISSION"] + method.lower()
 879
 880     @staticmethod
 881     def _set_location_header(main_topic, version, topic, id):
 882         """
 883         Insert response header Location with the URL of created item base on URL params
 884         :param main_topic:
 885         :param version:
 886         :param topic:
 887         :param id:
 888         :return: None
 889         """
 890         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 891         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 892         return
 893
 894     @staticmethod
 895     def _extract_query_string_operations(kwargs, method):
 896         """
 897
 898         :param kwargs:
 899         :return:
 900         """
 901         query_string_operations = []
 902         if kwargs:
 903             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 904                 if qs in kwargs and kwargs[qs].lower() != "false":
 905                     query_string_operations.append(qs.lower() + ":" + method.lower())
 906         return query_string_operations
 907
 908     @staticmethod
 909     def _manage_admin_query(token_info, kwargs, method, _id):
 910         """
 911         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 912         Check that users has rights to use them and returs the admin_query
 913         :param token_info: token_info rights obtained by token
 914         :param kwargs: query string input.
 915         :param method: http method: GET, POSST, PUT, ...
 916         :param _id:
 917         :return: admin_query dictionary with keys:
 918             public: True, False or None
 919             force: True or False
 920             project_id: tuple with projects used for accessing an element
 921             set_project: tuple with projects that a created element will belong to
 922             method: show, list, delete, write
 923         """
 924         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 925                        "admin": token_info["admin"], "public": None,
 926                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 927         if kwargs:
 928             # FORCE
 929             if "FORCE" in kwargs:
 930                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 931                     admin_query["force"] = True
 932                 del kwargs["FORCE"]
 933             # PUBLIC
 934             if "PUBLIC" in kwargs:
 935                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 936                     admin_query["public"] = True
 937                 else:
 938                     admin_query["public"] = False
 939                 del kwargs["PUBLIC"]
 940             # ADMIN
 941             if "ADMIN" in kwargs:
 942                 behave_as = kwargs.pop("ADMIN")
 943                 if behave_as.lower() != "false":
 944                     if not token_info["admin"]:
 945                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 946                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 947                         admin_query["project_id"] = ()
 948                     elif isinstance(behave_as, (list, tuple)):
 949                         admin_query["project_id"] = behave_as
 950                     else:   # isinstance(behave_as, str)
 951                         admin_query["project_id"] = (behave_as, )
 952             if "SET_PROJECT" in kwargs:
 953                 set_project = kwargs.pop("SET_PROJECT")
 954                 if not set_project:
 955                     admin_query["set_project"] = list(admin_query["project_id"])
 956                 else:
 957                     if isinstance(set_project, str):
 958                         set_project = (set_project, )
 959                     if admin_query["project_id"]:
 960                         for p in set_project:
 961                             if p not in admin_query["project_id"]:
 962                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 963                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 964                     admin_query["set_project"] = set_project
 965
 966             # PROJECT_READ
 967             # if "PROJECT_READ" in kwargs:
 968             #     admin_query["project"] = kwargs.pop("project")
 969             #     if admin_query["project"] == token_info["project_id"]:
 970         if method == "GET":
 971             if _id:
 972                 admin_query["method"] = "show"
 973             else:
 974                 admin_query["method"] = "list"
 975         elif method == "DELETE":
 976             admin_query["method"] = "delete"
 977         else:
 978             admin_query["method"] = "write"
 979         return admin_query
 980
 981     @cherrypy.expose
 982     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 983         token_info = None
 984         outdata = None
 985         _format = None
 986         method = "DONE"
 987         engine_topic = None
 988         rollback = []
 989         engine_session = None
 990         try:
 991             if not main_topic or not version or not topic:
 992                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 993                                    HTTPStatus.METHOD_NOT_ALLOWED)
 994             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 995                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 996                                    HTTPStatus.METHOD_NOT_ALLOWED)
 997             if version != 'v1':
 998                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 999
1000             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
1001                 method = kwargs.pop("METHOD")
1002             else:
1003                 method = cherrypy.request.method
1004
1005             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
1006             query_string_operations = self._extract_query_string_operations(kwargs, method)
1007             if main_topic == "admin" and topic == "tokens":
1008                 return self.token(method, _id, kwargs)
1009             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
1010             if main_topic == "admin" and topic == "domains":
1011                 return self.domain()
1012             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
1013             indata = self._format_in(kwargs)
1014             engine_topic = topic
1015
1016             if item and topic != "pm_jobs":
1017                 engine_topic = item
1018
1019             if main_topic == "nsd":
1020                 engine_topic = "nsds"
1021             elif main_topic == "vnfpkgm":
1022                 engine_topic = "vnfds"
1023                 if topic == "vnfpkg_op_occs":
1024                     engine_topic = "vnfpkgops"
1025                 if topic == "vnf_packages" and item == "action":
1026                     engine_topic = "vnfpkgops"
1027             elif main_topic == "nslcm":
1028                 engine_topic = "nsrs"
1029                 if topic == "ns_lcm_op_occs":
1030                     engine_topic = "nslcmops"
1031                 if topic == "vnfrs" or topic == "vnf_instances":
1032                     engine_topic = "vnfrs"
1033             elif main_topic == "nst":
1034                 engine_topic = "nsts"
1035             elif main_topic == "nsilcm":
1036                 engine_topic = "nsis"
1037                 if topic == "nsi_lcm_op_occs":
1038                     engine_topic = "nsilcmops"
1039             elif main_topic == "pdu":
1040                 engine_topic = "pdus"
1041             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
1042                 engine_topic = "vim_accounts"
1043
1044             if topic == "subscriptions":
1045                 engine_topic = main_topic + "_" + topic
1046
1047             if method == "GET":
1048                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
1049                     if item in ("vnfd", "nsd", "nst"):
1050                         path = "$DESCRIPTOR"
1051                     elif args:
1052                         path = args
1053                     elif item == "artifacts":
1054                         path = ()
1055                     else:
1056                         path = None
1057                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
1058                                                          cherrypy.request.headers.get("Accept"))
1059                     outdata = file
1060                 elif not _id:
1061                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
1062                 else:
1063                     if item == "reports":
1064                         # TODO check that project_id (_id in this context) has permissions
1065                         _id = args[0]
1066                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
1067
1068             elif method == "POST":
1069                 cherrypy.response.status = HTTPStatus.CREATED.value
1070                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1071                     _id = cherrypy.request.headers.get("Transaction-Id")
1072                     if not _id:
1073                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1074                                                       cherrypy.request.headers)
1075                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1076                                                            cherrypy.request.headers)
1077                     if completed:
1078                         self._set_location_header(main_topic, version, topic, _id)
1079                     else:
1080                         cherrypy.response.headers["Transaction-Id"] = _id
1081                     outdata = {"id": _id}
1082                 elif topic == "ns_instances_content":
1083                     # creates NSR
1084                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1085                     # creates nslcmop
1086                     indata["lcmOperationType"] = "instantiate"
1087                     indata["nsInstanceId"] = _id
1088                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1089                     self._set_location_header(main_topic, version, topic, _id)
1090                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1091                 elif topic == "ns_instances" and item:
1092                     indata["lcmOperationType"] = item
1093                     indata["nsInstanceId"] = _id
1094                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1095                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1096                     outdata = {"id": _id}
1097                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1098                 elif topic == "netslice_instances_content":
1099                     # creates NetSlice_Instance_record (NSIR)
1100                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1101                     self._set_location_header(main_topic, version, topic, _id)
1102                     indata["lcmOperationType"] = "instantiate"
1103                     indata["netsliceInstanceId"] = _id
1104                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1105                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1106                 elif topic == "netslice_instances" and item:
1107                     indata["lcmOperationType"] = item
1108                     indata["netsliceInstanceId"] = _id
1109                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1110                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1111                     outdata = {"id": _id}
1112                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1113                 elif topic == "vnf_packages" and item == "action":
1114                     indata["lcmOperationType"] = item
1115                     indata["vnfPkgId"] = _id
1116                     _id, _ = self.engine.new_item(rollback, engine_session, "vnfpkgops", indata, kwargs)
1117                     self._set_location_header(main_topic, version, "vnfpkg_op_occs", _id)
1118                     outdata = {"id": _id}
1119                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1120                 elif topic == "subscriptions":
1121                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1122                     self._set_location_header(main_topic, version, topic, _id)
1123                     link = {}
1124                     link["self"] = cherrypy.response.headers["Location"]
1125                     outdata = {"id": _id, "filter": indata["filter"], "callbackUri": indata["CallbackUri"],
1126                                "_links": link}
1127                     cherrypy.response.status = HTTPStatus.CREATED.value
1128                 else:
1129                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1130                                                       cherrypy.request.headers)
1131                     self._set_location_header(main_topic, version, topic, _id)
1132                     outdata = {"id": _id}
1133                     if op_id:
1134                         outdata["op_id"] = op_id
1135                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1136                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1137
1138             elif method == "DELETE":
1139                 if not _id:
1140                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1141                     cherrypy.response.status = HTTPStatus.OK.value
1142                 else:  # len(args) > 1
1143                     # for NS NSI generate an operation
1144                     op_id = None
1145                     if topic == "ns_instances_content" and not engine_session["force"]:
1146                         nslcmop_desc = {
1147                             "lcmOperationType": "terminate",
1148                             "nsInstanceId": _id,
1149                             "autoremove": True
1150                         }
1151                         op_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, kwargs)
1152                         if op_id:
1153                             outdata = {"_id": op_id}
1154                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1155                         nsilcmop_desc = {
1156                             "lcmOperationType": "terminate",
1157                             "netsliceInstanceId": _id,
1158                             "autoremove": True
1159                         }
1160                         op_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1161                         if op_id:
1162                             outdata = {"_id": op_id}
1163                     # if there is not any deletion in process, delete
1164                     if not op_id:
1165                         op_id = self.engine.del_item(engine_session, engine_topic, _id)
1166                         if op_id:
1167                             outdata = {"op_id": op_id}
1168                     cherrypy.response.status = HTTPStatus.ACCEPTED.value if op_id else HTTPStatus.NO_CONTENT.value
1169
1170             elif method in ("PUT", "PATCH"):
1171                 op_id = None
1172                 if not indata and not kwargs and not engine_session.get("set_project"):
1173                     raise NbiException("Nothing to update. Provide payload and/or query string",
1174                                        HTTPStatus.BAD_REQUEST)
1175                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1176                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1177                                                            cherrypy.request.headers)
1178                     if not completed:
1179                         cherrypy.response.headers["Transaction-Id"] = id
1180                 else:
1181                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1182
1183                 if op_id:
1184                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1185                     outdata = {"op_id": op_id}
1186                 else:
1187                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1188                     outdata = None
1189             else:
1190                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1191
1192             # if Role information changes, it is needed to reload the information of roles
1193             if topic == "roles" and method != "GET":
1194                 self.authenticator.load_operation_to_allowed_roles()
1195
1196             if topic == "projects" and method == "DELETE" \
1197                     or topic in ["users", "roles"] and method in ["PUT", "PATCH", "DELETE"]:
1198                 self.authenticator.remove_token_from_cache()
1199
1200             return self._format_out(outdata, token_info, _format)
1201         except Exception as e:
1202             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1203                               ValidationError, AuthconnException)):
1204                 http_code_value = cherrypy.response.status = e.http_code.value
1205                 http_code_name = e.http_code.name
1206                 cherrypy.log("Exception {}".format(e))
1207             else:
1208                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1209                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1210                 http_code_name = HTTPStatus.BAD_REQUEST.name
1211             if hasattr(outdata, "close"):  # is an open file
1212                 outdata.close()
1213             error_text = str(e)
1214             rollback.reverse()
1215             for rollback_item in rollback:
1216                 try:
1217                     if rollback_item.get("operation") == "set":
1218                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1219                                                rollback_item["content"], fail_on_empty=False)
1220                     elif rollback_item.get("operation") == "del_list":
1221                         self.engine.db.del_list(rollback_item["topic"], rollback_item["filter"],
1222                                                 fail_on_empty=False)
1223                     else:
1224                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1225                                                fail_on_empty=False)
1226                 except Exception as e2:
1227                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1228                     cherrypy.log(rollback_error_text)
1229                     error_text += ". " + rollback_error_text
1230             # if isinstance(e, MsgException):
1231             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1232             #         engine_topic[:-1], method, error_text)
1233             problem_details = {
1234                 "code": http_code_name,
1235                 "status": http_code_value,
1236                 "detail": error_text,
1237             }
1238             return self._format_out(problem_details, token_info)
1239             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1240         finally:
1241             if token_info:
1242                 self._format_login(token_info)
1243                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1244                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1245                         if outdata.get(logging_id):
1246                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1247
1248
1249 def _start_service():
1250     """
1251     Callback function called when cherrypy.engine starts
1252     Override configuration with env variables
1253     Set database, storage, message configuration
1254     Init database with admin/admin user password
1255     """
1256     global nbi_server
1257     global subscription_thread
1258     cherrypy.log.error("Starting osm_nbi")
1259     # update general cherrypy configuration
1260     update_dict = {}
1261
1262     engine_config = cherrypy.tree.apps['/osm'].config
1263     for k, v in environ.items():
1264         if not k.startswith("OSMNBI_"):
1265             continue
1266         k1, _, k2 = k[7:].lower().partition("_")
1267         if not k2:
1268             continue
1269         try:
1270             # update static configuration
1271             if k == 'OSMNBI_STATIC_DIR':
1272                 engine_config["/static"]['tools.staticdir.dir'] = v
1273                 engine_config["/static"]['tools.staticdir.on'] = True
1274             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1275                 update_dict['server.socket_port'] = int(v)
1276             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1277                 update_dict['server.socket_host'] = v
1278             elif k1 in ("server", "test", "auth", "log"):
1279                 update_dict[k1 + '.' + k2] = v
1280             elif k1 in ("message", "database", "storage", "authentication"):
1281                 # k2 = k2.replace('_', '.')
1282                 if k2 in ("port", "db_port"):
1283                     engine_config[k1][k2] = int(v)
1284                 else:
1285                     engine_config[k1][k2] = v
1286
1287         except ValueError as e:
1288             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1289         except Exception as e:
1290             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1291
1292     if update_dict:
1293         cherrypy.config.update(update_dict)
1294         engine_config["global"].update(update_dict)
1295
1296     # logging cherrypy
1297     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1298     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1299     logger_server = logging.getLogger("cherrypy.error")
1300     logger_access = logging.getLogger("cherrypy.access")
1301     logger_cherry = logging.getLogger("cherrypy")
1302     logger_nbi = logging.getLogger("nbi")
1303
1304     if "log.file" in engine_config["global"]:
1305         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1306                                                             maxBytes=100e6, backupCount=9, delay=0)
1307         file_handler.setFormatter(log_formatter_simple)
1308         logger_cherry.addHandler(file_handler)
1309         logger_nbi.addHandler(file_handler)
1310     # log always to standard output
1311     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1312                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1313                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1314                             }.items():
1315         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1316         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1317         str_handler = logging.StreamHandler()
1318         str_handler.setFormatter(log_formatter_cherry)
1319         logger.addHandler(str_handler)
1320
1321     if engine_config["global"].get("log.level"):
1322         logger_cherry.setLevel(engine_config["global"]["log.level"])
1323         logger_nbi.setLevel(engine_config["global"]["log.level"])
1324
1325     # logging other modules
1326     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1327         engine_config[k1]["logger_name"] = logname
1328         logger_module = logging.getLogger(logname)
1329         if "logfile" in engine_config[k1]:
1330             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1331                                                                 maxBytes=100e6, backupCount=9, delay=0)
1332             file_handler.setFormatter(log_formatter_simple)
1333             logger_module.addHandler(file_handler)
1334         if "loglevel" in engine_config[k1]:
1335             logger_module.setLevel(engine_config[k1]["loglevel"])
1336     # TODO add more entries, e.g.: storage
1337     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1338     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1339     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1340     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1341
1342     # start subscriptions thread:
1343     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1344     subscription_thread.start()
1345     # Do not capture except SubscriptionException
1346
1347     backend = engine_config["authentication"]["backend"]
1348     cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
1349                        .format(nbi_version, nbi_version_date, backend))
1350
1351
1352 def _stop_service():
1353     """
1354     Callback function called when cherrypy.engine stops
1355     TODO: Ending database connections.
1356     """
1357     global subscription_thread
1358     if subscription_thread:
1359         subscription_thread.terminate()
1360     subscription_thread = None
1361     cherrypy.tree.apps['/osm'].root.engine.stop()
1362     cherrypy.log.error("Stopping osm_nbi")
1363
1364
1365 def nbi(config_file):
1366     global nbi_server
1367     # conf = {
1368     #     '/': {
1369     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1370     #         'tools.sessions.on': True,
1371     #         'tools.response_headers.on': True,
1372     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1373     #     }
1374     # }
1375     # cherrypy.Server.ssl_module = 'builtin'
1376     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1377     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1378     # cherrypy.Server.thread_pool = 10
1379     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1380
1381     # cherrypy.config.update({'tools.auth_basic.on': True,
1382     #    'tools.auth_basic.realm': 'localhost',
1383     #    'tools.auth_basic.checkpassword': validate_password})
1384     nbi_server = Server()
1385     cherrypy.engine.subscribe('start', _start_service)
1386     cherrypy.engine.subscribe('stop', _stop_service)
1387     cherrypy.quickstart(nbi_server, '/osm', config_file)
1388
1389
1390 def usage():
1391     print("""Usage: {} [options]
1392         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1393         -h|--help: shows this help
1394         """.format(sys.argv[0]))
1395     # --log-socket-host HOST: send logs to this host")
1396     # --log-socket-port PORT: send logs using this port (default: 9022)")
1397
1398
1399 if __name__ == '__main__':
1400     try:
1401         # load parameters and configuration
1402         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1403         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1404         config_file = None
1405         for o, a in opts:
1406             if o in ("-h", "--help"):
1407                 usage()
1408                 sys.exit()
1409             elif o in ("-c", "--config"):
1410                 config_file = a
1411             # elif o == "--log-socket-port":
1412             #     log_socket_port = a
1413             # elif o == "--log-socket-host":
1414             #     log_socket_host = a
1415             # elif o == "--log-file":
1416             #     log_file = a
1417             else:
1418                 assert False, "Unhandled option"
1419         if config_file:
1420             if not path.isfile(config_file):
1421                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1422                 exit(1)
1423         else:
1424             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1425                 if path.isfile(config_file):
1426                     break
1427             else:
1428                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1429                 exit(1)
1430         nbi(config_file)
1431     except getopt.GetoptError as e:
1432         print(str(e), file=sys.stderr)
1433         # usage()
1434         exit(1)