osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import osm_nbi.html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from osm_nbi.authconn import AuthException, AuthconnException
  28 from osm_nbi.auth import Authenticator
  29 from osm_nbi.engine import Engine, EngineException
  30 from osm_nbi.subscriptions import SubscriptionThread
  31 from osm_nbi.validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38 from osm_nbi import version as nbi_version, version_date as nbi_version_date
  39
  40 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  41
  42 __version__ = "0.1.3"    # file version, not NBI version
  43 version_date = "Aug 2019"
  44
  45 database_version = '1.2'
  46 auth_database_version = '1.0'
  47 nbi_server = None           # instance of Server class
  48 subscription_thread = None  # instance of SubscriptionThread class
  49
  50
  51 """
  52 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  53 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  54         /nsd/v1
  55             /ns_descriptors_content                             O       O
  56                 /<nsdInfoId>                                    O       O       O       O
  57             /ns_descriptors                                     O5      O5
  58                 /<nsdInfoId>                                    O5                      O5      5
  59                     /nsd_content                                O5              O5
  60                     /nsd                                        O
  61                     /artifacts[/<artifactPath>]                 O
  62             /pnf_descriptors                                    5       5
  63                 /<pnfdInfoId>                                   5                       5       5
  64                     /pnfd_content                               5               5
  65             /subscriptions                                      5       5
  66                 /<subscriptionId>                               5                       X
  67
  68         /vnfpkgm/v1
  69             /vnf_packages_content                               O       O
  70                 /<vnfPkgId>                                     O                       O
  71             /vnf_packages                                       O5      O5
  72                 /<vnfPkgId>                                     O5                      O5      5
  73                     /package_content                            O5               O5
  74                         /upload_from_uri                                X
  75                     /vnfd                                       O5
  76                     /artifacts[/<artifactPath>]                 O5
  77             /subscriptions                                      X       X
  78                 /<subscriptionId>                               X                       X
  79
  80         /nslcm/v1
  81             /ns_instances_content                               O       O
  82                 /<nsInstanceId>                                 O                       O
  83             /ns_instances                                       5       5
  84                 /<nsInstanceId>                                 O5                      O5
  85                     instantiate                                         O5
  86                     terminate                                           O5
  87                     action                                              O
  88                     scale                                               O5
  89                     heal                                                5
  90             /ns_lcm_op_occs                                     5       5
  91                 /<nsLcmOpOccId>                                 5                       5       5
  92                     TO BE COMPLETED                             5               5
  93             /vnf_instances  (also vnfrs for compatibility)      O
  94                 /<vnfInstanceId>                                O
  95             /subscriptions                                      5       5
  96                 /<subscriptionId>                               5                       X
  97
  98         /pdu/v1
  99             /pdu_descriptors                                    O       O
 100                 /<id>                                           O               O       O       O
 101
 102         /admin/v1
 103             /tokens                                             O       O
 104                 /<id>                                           O                       O
 105             /users                                              O       O
 106                 /<id>                                           O               O       O       O
 107             /projects                                           O       O
 108                 /<id>                                           O                       O
 109             /vim_accounts  (also vims for compatibility)        O       O
 110                 /<id>                                           O                       O       O
 111             /wim_accounts                                       O       O
 112                 /<id>                                           O                       O       O
 113             /sdns                                               O       O
 114                 /<id>                                           O                       O       O
 115
 116         /nst/v1                                                 O       O
 117             /netslice_templates_content                         O       O
 118                 /<nstInfoId>                                    O       O       O       O
 119             /netslice_templates                                 O       O
 120                 /<nstInfoId>                                    O                       O       O
 121                     /nst_content                                O               O
 122                     /nst                                        O
 123                     /artifacts[/<artifactPath>]                 O
 124             /subscriptions                                      X       X
 125                 /<subscriptionId>                               X                       X
 126
 127         /nsilcm/v1
 128             /netslice_instances_content                         O       O
 129                 /<SliceInstanceId>                              O                       O
 130             /netslice_instances                                 O       O
 131                 /<SliceInstanceId>                              O                       O
 132                     instantiate                                         O
 133                     terminate                                           O
 134                     action                                              O
 135             /nsi_lcm_op_occs                                    O       O
 136                 /<nsiLcmOpOccId>                                O                       O       O
 137             /subscriptions                                      X       X
 138                 /<subscriptionId>                               X                       X
 139
 140 query string:
 141     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 142         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 143         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 144         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 145         attrName := string
 146     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 147     item of the array, that is, pass if any item of the array pass the filter.
 148     It allows both ne and neq for not equal
 149     TODO: 4.3.3 Attribute selectors
 150         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 151         (none)  … same as “exclude_default”
 152         all_fields      … all attributes.
 153         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 154         conditionally mandatory, and that are not provided in <list>.
 155         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 156         are not conditionally mandatory, and that are provided in <list>.
 157         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 158         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 159         the particular resource
 160         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 161         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 162         present specification for the particular resource, but that are not part of <list>
 163     Additionally it admits some administrator values:
 164         FORCE: To force operations skipping dependency checkings
 165         ADMIN: To act as an administrator or a different project
 166         PUBLIC: To get public descriptors or set a descriptor as public
 167         SET_PROJECT: To make a descriptor available for other project
 168
 169 Header field name       Reference       Example Descriptions
 170     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 171     This header field shall be present if the response is expected to have a non-empty message body.
 172     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 173     This header field shall be present if the request has a non-empty message body.
 174     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 175     Details are specified in clause 4.5.3.
 176     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 177 Header field name       Reference       Example Descriptions
 178     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 179     This header field shall be present if the response has a non-empty message body.
 180     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 181     new resource has been created.
 182     This header field shall be present if the response status code is 201 or 3xx.
 183     In the present document this header field is also used if the response status code is 202 and a new resource was
 184     created.
 185     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 186     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 187     token.
 188     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 189     certain resources.
 190     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 191     response, and the total length of the file.
 192     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 193 """
 194
 195 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 196 # ^ Contains possible administrative query string words:
 197 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 198 #           (not owned by my session project).
 199 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 200 #     FORCE=True(by default)|False: Force edition/deletion operations
 201 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 202
 203 valid_url_methods = {
 204     # contains allowed URL and methods, and the role_permission name
 205     "admin": {
 206         "v1": {
 207             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 208                        "ROLE_PERMISSION": "tokens:",
 209                        "<ID>": {"METHODS": ("GET", "DELETE"),
 210                                 "ROLE_PERMISSION": "tokens:id:"
 211                                 }
 212                        },
 213             "users": {"METHODS": ("GET", "POST"),
 214                       "ROLE_PERMISSION": "users:",
 215                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 216                                "ROLE_PERMISSION": "users:id:"
 217                                }
 218                       },
 219             "projects": {"METHODS": ("GET", "POST"),
 220                          "ROLE_PERMISSION": "projects:",
 221                          "<ID>": {"METHODS": ("GET", "DELETE", "PUT"),
 222                                   "ROLE_PERMISSION": "projects:id:"}
 223                          },
 224             "roles": {"METHODS": ("GET", "POST"),
 225                       "ROLE_PERMISSION": "roles:",
 226                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PUT"),
 227                                "ROLE_PERMISSION": "roles:id:"
 228                                }
 229                       },
 230             "vims": {"METHODS": ("GET", "POST"),
 231                      "ROLE_PERMISSION": "vims:",
 232                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 233                               "ROLE_PERMISSION": "vims:id:"
 234                               }
 235                      },
 236             "vim_accounts": {"METHODS": ("GET", "POST"),
 237                              "ROLE_PERMISSION": "vim_accounts:",
 238                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 239                                       "ROLE_PERMISSION": "vim_accounts:id:"
 240                                       }
 241                              },
 242             "wim_accounts": {"METHODS": ("GET", "POST"),
 243                              "ROLE_PERMISSION": "wim_accounts:",
 244                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 245                                       "ROLE_PERMISSION": "wim_accounts:id:"
 246                                       }
 247                              },
 248             "sdns": {"METHODS": ("GET", "POST"),
 249                      "ROLE_PERMISSION": "sdn_controllers:",
 250                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 251                               "ROLE_PERMISSION": "sdn_controllers:id:"
 252                               }
 253                      },
 254         }
 255     },
 256     "pdu": {
 257         "v1": {
 258             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 259                                 "ROLE_PERMISSION": "pduds:",
 260                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 261                                          "ROLE_PERMISSION": "pduds:id:"
 262                                          }
 263                                 },
 264         }
 265     },
 266     "nsd": {
 267         "v1": {
 268             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 269                                        "ROLE_PERMISSION": "nsds:",
 270                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 271                                                 "ROLE_PERMISSION": "nsds:id:"
 272                                                 }
 273                                        },
 274             "ns_descriptors": {"METHODS": ("GET", "POST"),
 275                                "ROLE_PERMISSION": "nsds:",
 276                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 277                                         "ROLE_PERMISSION": "nsds:id:",
 278                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 279                                                         "ROLE_PERMISSION": "nsds:id:content:",
 280                                                         },
 281                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 282                                                 "ROLE_PERMISSION": "nsds:id:content:"
 283                                                 },
 284                                         "artifacts": {"*": {"METHODS": ("GET",),
 285                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 286                                                             }
 287                                                       }
 288                                         }
 289                                },
 290             "pnf_descriptors": {"TODO": ("GET", "POST"),
 291                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 292                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 293                                          }
 294                                 },
 295             "subscriptions": {"TODO": ("GET", "POST"),
 296                               "<ID>": {"TODO": ("GET", "DELETE")}
 297                               },
 298         }
 299     },
 300     "vnfpkgm": {
 301         "v1": {
 302             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 303                                      "ROLE_PERMISSION": "vnfds:",
 304                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 305                                               "ROLE_PERMISSION": "vnfds:id:"}
 306                                      },
 307             "vnf_packages": {"METHODS": ("GET", "POST"),
 308                              "ROLE_PERMISSION": "vnfds:",
 309                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 310                                       "ROLE_PERMISSION": "vnfds:id:",
 311                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 312                                                           "ROLE_PERMISSION": "vnfds:id:",
 313                                                           "upload_from_uri": {"METHODS": (),
 314                                                                               "TODO": ("POST", ),
 315                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 316                                                                               }
 317                                                           },
 318                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 319                                                "ROLE_PERMISSION": "vnfds:id:content:"
 320                                                },
 321                                       "artifacts": {"*": {"METHODS": ("GET", ),
 322                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 323                                                           }
 324                                                     }
 325                                       }
 326                              },
 327             "subscriptions": {"TODO": ("GET", "POST"),
 328                               "<ID>": {"TODO": ("GET", "DELETE")}
 329                               },
 330         }
 331     },
 332     "nslcm": {
 333         "v1": {
 334             "ns_instances_content": {"METHODS": ("GET", "POST"),
 335                                      "ROLE_PERMISSION": "ns_instances:",
 336                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 337                                               "ROLE_PERMISSION": "ns_instances:id:"
 338                                               }
 339                                      },
 340             "ns_instances": {"METHODS": ("GET", "POST"),
 341                              "ROLE_PERMISSION": "ns_instances:",
 342                              "<ID>": {"METHODS": ("GET", "DELETE"),
 343                                       "ROLE_PERMISSION": "ns_instances:id:",
 344                                       "scale": {"METHODS": ("POST",),
 345                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 346                                                 },
 347                                       "terminate": {"METHODS": ("POST",),
 348                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 349                                                     },
 350                                       "instantiate": {"METHODS": ("POST",),
 351                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 352                                                       },
 353                                       "action": {"METHODS": ("POST",),
 354                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 355                                                  },
 356                                       }
 357                              },
 358             "ns_lcm_op_occs": {"METHODS": ("GET",),
 359                                "ROLE_PERMISSION": "ns_instances:opps:",
 360                                "<ID>": {"METHODS": ("GET",),
 361                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 362                                         },
 363                                },
 364             "vnfrs": {"METHODS": ("GET",),
 365                       "ROLE_PERMISSION": "vnf_instances:",
 366                       "<ID>": {"METHODS": ("GET",),
 367                                "ROLE_PERMISSION": "vnf_instances:id:"
 368                                }
 369                       },
 370             "vnf_instances": {"METHODS": ("GET",),
 371                               "ROLE_PERMISSION": "vnf_instances:",
 372                               "<ID>": {"METHODS": ("GET",),
 373                                        "ROLE_PERMISSION": "vnf_instances:id:"
 374                                        }
 375                               },
 376         }
 377     },
 378     "nst": {
 379         "v1": {
 380             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 381                                            "ROLE_PERMISSION": "slice_templates:",
 382                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 383                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 384                                            },
 385             "netslice_templates": {"METHODS": ("GET", "POST"),
 386                                    "ROLE_PERMISSION": "slice_templates:",
 387                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 388                                             "TODO": ("PATCH",),
 389                                             "ROLE_PERMISSION": "slice_templates:id:",
 390                                             "nst_content": {"METHODS": ("GET", "PUT"),
 391                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 392                                                             },
 393                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 394                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 395                                                     },
 396                                             "artifacts": {"*": {"METHODS": ("GET",),
 397                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 398                                                                 }
 399                                                           }
 400                                             }
 401                                    },
 402             "subscriptions": {"TODO": ("GET", "POST"),
 403                               "<ID>": {"TODO": ("GET", "DELETE")}
 404                               },
 405         }
 406     },
 407     "nsilcm": {
 408         "v1": {
 409             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 410                                            "ROLE_PERMISSION": "slice_instances:",
 411                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 412                                                     "ROLE_PERMISSION": "slice_instances:id:"
 413                                                     }
 414                                            },
 415             "netslice_instances": {"METHODS": ("GET", "POST"),
 416                                    "ROLE_PERMISSION": "slice_instances:",
 417                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 418                                             "ROLE_PERMISSION": "slice_instances:id:",
 419                                             "terminate": {"METHODS": ("POST",),
 420                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 421                                                           },
 422                                             "instantiate": {"METHODS": ("POST",),
 423                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 424                                                             },
 425                                             "action": {"METHODS": ("POST",),
 426                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 427                                                        },
 428                                             }
 429                                    },
 430             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 431                                 "ROLE_PERMISSION": "slice_instances:opps:",
 432                                 "<ID>": {"METHODS": ("GET",),
 433                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 434                                          },
 435                                 },
 436         }
 437     },
 438     "nspm": {
 439         "v1": {
 440             "pm_jobs": {
 441                 "<ID>": {
 442                     "reports": {
 443                         "<ID>": {"METHODS": ("GET",),
 444                                  "ROLE_PERMISSION": "reports:id:",
 445                                  }
 446                     }
 447                 },
 448             },
 449         },
 450     },
 451 }
 452
 453
 454 class NbiException(Exception):
 455
 456     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 457         Exception.__init__(self, message)
 458         self.http_code = http_code
 459
 460
 461 class Server(object):
 462     instance = 0
 463     # to decode bytes to str
 464     reader = getreader("utf-8")
 465
 466     def __init__(self):
 467         self.instance += 1
 468         self.engine = Engine()
 469         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 470
 471     def _format_in(self, kwargs):
 472         try:
 473             indata = None
 474             if cherrypy.request.body.length:
 475                 error_text = "Invalid input format "
 476
 477                 if "Content-Type" in cherrypy.request.headers:
 478                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 479                         error_text = "Invalid json format "
 480                         indata = json.load(self.reader(cherrypy.request.body))
 481                         cherrypy.request.headers.pop("Content-File-MD5", None)
 482                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 483                         error_text = "Invalid yaml format "
 484                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 485                         cherrypy.request.headers.pop("Content-File-MD5", None)
 486                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 487                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 488                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 489                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 490                         indata = cherrypy.request.body  # .read()
 491                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 492                         if "descriptor_file" in kwargs:
 493                             filecontent = kwargs.pop("descriptor_file")
 494                             if not filecontent.file:
 495                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 496                             indata = filecontent.file  # .read()
 497                             if filecontent.content_type.value:
 498                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 499                     else:
 500                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 501                         #                          "Only 'Content-Type' of type 'application/json' or
 502                         # 'application/yaml' for input format are available")
 503                         error_text = "Invalid yaml format "
 504                         indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 505                         cherrypy.request.headers.pop("Content-File-MD5", None)
 506                 else:
 507                     error_text = "Invalid yaml format "
 508                     indata = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 509                     cherrypy.request.headers.pop("Content-File-MD5", None)
 510             if not indata:
 511                 indata = {}
 512
 513             format_yaml = False
 514             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 515                 format_yaml = True
 516
 517             for k, v in kwargs.items():
 518                 if isinstance(v, str):
 519                     if v == "":
 520                         kwargs[k] = None
 521                     elif format_yaml:
 522                         try:
 523                             kwargs[k] = yaml.load(v, Loader=yaml.SafeLoader)
 524                         except Exception:
 525                             pass
 526                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 527                         try:
 528                             kwargs[k] = int(v)
 529                         except Exception:
 530                             try:
 531                                 kwargs[k] = float(v)
 532                             except Exception:
 533                                 pass
 534                     elif v.find(",") > 0:
 535                         kwargs[k] = v.split(",")
 536                 elif isinstance(v, (list, tuple)):
 537                     for index in range(0, len(v)):
 538                         if v[index] == "":
 539                             v[index] = None
 540                         elif format_yaml:
 541                             try:
 542                                 v[index] = yaml.load(v[index], Loader=yaml.SafeLoader)
 543                             except Exception:
 544                                 pass
 545
 546             return indata
 547         except (ValueError, yaml.YAMLError) as exc:
 548             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 549         except KeyError as exc:
 550             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 551         except Exception as exc:
 552             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 553
 554     @staticmethod
 555     def _format_out(data, token_info=None, _format=None):
 556         """
 557         return string of dictionary data according to requested json, yaml, xml. By default json
 558         :param data: response to be sent. Can be a dict, text or file
 559         :param token_info: Contains among other username and project
 560         :param _format: The format to be set as Content-Type if data is a file
 561         :return: None
 562         """
 563         accept = cherrypy.request.headers.get("Accept")
 564         if data is None:
 565             if accept and "text/html" in accept:
 566                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 567             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 568             return
 569         elif hasattr(data, "read"):  # file object
 570             if _format:
 571                 cherrypy.response.headers["Content-Type"] = _format
 572             elif "b" in data.mode:  # binariy asssumig zip
 573                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 574             else:
 575                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 576             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 577             return data
 578         if accept:
 579             if "application/json" in accept:
 580                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 581                 a = json.dumps(data, indent=4) + "\n"
 582                 return a.encode("utf8")
 583             elif "text/html" in accept:
 584                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 585
 586             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 587                 pass
 588             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 589             elif cherrypy.response.status >= 400:
 590                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 591                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 592                                          "for output format are available")
 593         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 594         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 595                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 596
 597     @cherrypy.expose
 598     def index(self, *args, **kwargs):
 599         token_info = None
 600         try:
 601             if cherrypy.request.method == "GET":
 602                 token_info = self.authenticator.authorize()
 603                 outdata = token_info   # Home page
 604             else:
 605                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 606                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 607
 608             return self._format_out(outdata, token_info)
 609
 610         except (EngineException, AuthException) as e:
 611             # cherrypy.log("index Exception {}".format(e))
 612             cherrypy.response.status = e.http_code.value
 613             return self._format_out("Welcome to OSM!", token_info)
 614
 615     @cherrypy.expose
 616     def version(self, *args, **kwargs):
 617         # TODO consider to remove and provide version using the static version file
 618         try:
 619             if cherrypy.request.method != "GET":
 620                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 621             elif args or kwargs:
 622                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 623             # TODO include version of other modules, pick up from some kafka admin message
 624             osm_nbi_version = {"version": nbi_version, "date": nbi_version_date}
 625             return self._format_out(osm_nbi_version)
 626         except NbiException as e:
 627             cherrypy.response.status = e.http_code.value
 628             problem_details = {
 629                 "code": e.http_code.name,
 630                 "status": e.http_code.value,
 631                 "detail": str(e),
 632             }
 633             return self._format_out(problem_details, None)
 634
 635     @staticmethod
 636     def _format_login(token_info):
 637         """
 638         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 639         log this information
 640         :param token_info: Dictionary with token content
 641         :return: None
 642         """
 643         cherrypy.request.login = token_info.get("username", "-")
 644         if token_info.get("project_name"):
 645             cherrypy.request.login += "/" + token_info["project_name"]
 646         if token_info.get("id"):
 647             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 648
 649     @cherrypy.expose
 650     def token(self, method, token_id=None, kwargs=None):
 651         token_info = None
 652         # self.engine.load_dbase(cherrypy.request.app.config)
 653         indata = self._format_in(kwargs)
 654         if not isinstance(indata, dict):
 655             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 656
 657         if method == "GET":
 658             token_info = self.authenticator.authorize()
 659             # for logging
 660             self._format_login(token_info)
 661             if token_id:
 662                 outdata = self.authenticator.get_token(token_info, token_id)
 663             else:
 664                 outdata = self.authenticator.get_token_list(token_info)
 665         elif method == "POST":
 666             try:
 667                 token_info = self.authenticator.authorize()
 668             except Exception:
 669                 token_info = None
 670             if kwargs:
 671                 indata.update(kwargs)
 672             # This is needed to log the user when authentication fails
 673             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 674             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 675             cherrypy.session['Authorization'] = outdata["_id"]
 676             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 677             # for logging
 678             self._format_login(token_info)
 679
 680             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 681             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 682         elif method == "DELETE":
 683             if not token_id and "id" in kwargs:
 684                 token_id = kwargs["id"]
 685             elif not token_id:
 686                 token_info = self.authenticator.authorize()
 687                 # for logging
 688                 self._format_login(token_info)
 689                 token_id = token_info["_id"]
 690             outdata = self.authenticator.del_token(token_id)
 691             token_info = None
 692             cherrypy.session['Authorization'] = "logout"
 693             # cherrypy.response.cookie["Authorization"] = token_id
 694             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 695         else:
 696             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 697         return self._format_out(outdata, token_info)
 698
 699     @cherrypy.expose
 700     def test(self, *args, **kwargs):
 701         thread_info = None
 702         if args and args[0] == "help":
 703             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 704                    "sleep/<time>\nmessage/topic\n</pre></html>"
 705
 706         elif args and args[0] == "init":
 707             try:
 708                 # self.engine.load_dbase(cherrypy.request.app.config)
 709                 self.engine.create_admin()
 710                 return "Done. User 'admin', password 'admin' created"
 711             except Exception:
 712                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 713                 return self._format_out("Database already initialized")
 714         elif args and args[0] == "file":
 715             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 716                                                   "text/plain", "attachment")
 717         elif args and args[0] == "file2":
 718             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 719             f = open(f_path, "r")
 720             cherrypy.response.headers["Content-type"] = "text/plain"
 721             return f
 722
 723         elif len(args) == 2 and args[0] == "db-clear":
 724             deleted_info = self.engine.db.del_list(args[1], kwargs)
 725             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 726         elif len(args) and args[0] == "fs-clear":
 727             if len(args) >= 2:
 728                 folders = (args[1],)
 729             else:
 730                 folders = self.engine.fs.dir_ls(".")
 731             for folder in folders:
 732                 self.engine.fs.file_delete(folder)
 733             return ",".join(folders) + " folders deleted\n"
 734         elif args and args[0] == "login":
 735             if not cherrypy.request.headers.get("Authorization"):
 736                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 737                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 738         elif args and args[0] == "login2":
 739             if not cherrypy.request.headers.get("Authorization"):
 740                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 741                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 742         elif args and args[0] == "sleep":
 743             sleep_time = 5
 744             try:
 745                 sleep_time = int(args[1])
 746             except Exception:
 747                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 748                 return self._format_out("Database already initialized")
 749             thread_info = cherrypy.thread_data
 750             print(thread_info)
 751             time.sleep(sleep_time)
 752             # thread_info
 753         elif len(args) >= 2 and args[0] == "message":
 754             main_topic = args[1]
 755             return_text = "<html><pre>{} ->\n".format(main_topic)
 756             try:
 757                 if cherrypy.request.method == 'POST':
 758                     to_send = yaml.load(cherrypy.request.body, Loader=yaml.SafeLoader)
 759                     for k, v in to_send.items():
 760                         self.engine.msg.write(main_topic, k, v)
 761                         return_text += "  {}: {}\n".format(k, v)
 762                 elif cherrypy.request.method == 'GET':
 763                     for k, v in kwargs.items():
 764                         self.engine.msg.write(main_topic, k, yaml.load(v), Loader=yaml.SafeLoader)
 765                         return_text += "  {}: {}\n".format(k, yaml.load(v), Loader=yaml.SafeLoader)
 766             except Exception as e:
 767                 return_text += "Error: " + str(e)
 768             return_text += "</pre></html>\n"
 769             return return_text
 770
 771         return_text = (
 772             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 773             "  kwargs: {}\n".format(kwargs) +
 774             "  headers: {}\n".format(cherrypy.request.headers) +
 775             "  path_info: {}\n".format(cherrypy.request.path_info) +
 776             "  query_string: {}\n".format(cherrypy.request.query_string) +
 777             "  session: {}\n".format(cherrypy.session) +
 778             "  cookie: {}\n".format(cherrypy.request.cookie) +
 779             "  method: {}\n".format(cherrypy.request.method) +
 780             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 781             "  body:\n")
 782         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 783         if cherrypy.request.body.length:
 784             return_text += "    content: {}\n".format(
 785                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 786         if thread_info:
 787             return_text += "thread: {}\n".format(thread_info)
 788         return_text += "</pre></html>"
 789         return return_text
 790
 791     @staticmethod
 792     def _check_valid_url_method(method, *args):
 793         if len(args) < 3:
 794             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 795
 796         reference = valid_url_methods
 797         for arg in args:
 798             if arg is None:
 799                 break
 800             if not isinstance(reference, dict):
 801                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 802                                    HTTPStatus.METHOD_NOT_ALLOWED)
 803
 804             if arg in reference:
 805                 reference = reference[arg]
 806             elif "<ID>" in reference:
 807                 reference = reference["<ID>"]
 808             elif "*" in reference:
 809                 reference = reference["*"]
 810                 break
 811             else:
 812                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 813         if "TODO" in reference and method in reference["TODO"]:
 814             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 815         elif "METHODS" in reference and method not in reference["METHODS"]:
 816             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 817         return reference["ROLE_PERMISSION"] + method.lower()
 818
 819     @staticmethod
 820     def _set_location_header(main_topic, version, topic, id):
 821         """
 822         Insert response header Location with the URL of created item base on URL params
 823         :param main_topic:
 824         :param version:
 825         :param topic:
 826         :param id:
 827         :return: None
 828         """
 829         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 830         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 831         return
 832
 833     @staticmethod
 834     def _extract_query_string_operations(kwargs, method):
 835         """
 836
 837         :param kwargs:
 838         :return:
 839         """
 840         query_string_operations = []
 841         if kwargs:
 842             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 843                 if qs in kwargs and kwargs[qs].lower() != "false":
 844                     query_string_operations.append(qs.lower() + ":" + method.lower())
 845         return query_string_operations
 846
 847     @staticmethod
 848     def _manage_admin_query(token_info, kwargs, method, _id):
 849         """
 850         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 851         Check that users has rights to use them and returs the admin_query
 852         :param token_info: token_info rights obtained by token
 853         :param kwargs: query string input.
 854         :param method: http method: GET, POSST, PUT, ...
 855         :param _id:
 856         :return: admin_query dictionary with keys:
 857             public: True, False or None
 858             force: True or False
 859             project_id: tuple with projects used for accessing an element
 860             set_project: tuple with projects that a created element will belong to
 861             method: show, list, delete, write
 862         """
 863         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 864                        "admin": token_info["admin"], "public": None,
 865                        "allow_show_user_project_role": token_info["allow_show_user_project_role"]}
 866         if kwargs:
 867             # FORCE
 868             if "FORCE" in kwargs:
 869                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 870                     admin_query["force"] = True
 871                 del kwargs["FORCE"]
 872             # PUBLIC
 873             if "PUBLIC" in kwargs:
 874                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 875                     admin_query["public"] = True
 876                 else:
 877                     admin_query["public"] = False
 878                 del kwargs["PUBLIC"]
 879             # ADMIN
 880             if "ADMIN" in kwargs:
 881                 behave_as = kwargs.pop("ADMIN")
 882                 if behave_as.lower() != "false":
 883                     if not token_info["admin"]:
 884                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 885                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 886                         admin_query["project_id"] = ()
 887                     elif isinstance(behave_as, (list, tuple)):
 888                         admin_query["project_id"] = behave_as
 889                     else:   # isinstance(behave_as, str)
 890                         admin_query["project_id"] = (behave_as, )
 891             if "SET_PROJECT" in kwargs:
 892                 set_project = kwargs.pop("SET_PROJECT")
 893                 if not set_project:
 894                     admin_query["set_project"] = list(admin_query["project_id"])
 895                 else:
 896                     if isinstance(set_project, str):
 897                         set_project = (set_project, )
 898                     if admin_query["project_id"]:
 899                         for p in set_project:
 900                             if p not in admin_query["project_id"]:
 901                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 902                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 903                     admin_query["set_project"] = set_project
 904
 905             # PROJECT_READ
 906             # if "PROJECT_READ" in kwargs:
 907             #     admin_query["project"] = kwargs.pop("project")
 908             #     if admin_query["project"] == token_info["project_id"]:
 909         if method == "GET":
 910             if _id:
 911                 admin_query["method"] = "show"
 912             else:
 913                 admin_query["method"] = "list"
 914         elif method == "DELETE":
 915             admin_query["method"] = "delete"
 916         else:
 917             admin_query["method"] = "write"
 918         return admin_query
 919
 920     @cherrypy.expose
 921     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 922         token_info = None
 923         outdata = None
 924         _format = None
 925         method = "DONE"
 926         engine_topic = None
 927         rollback = []
 928         engine_session = None
 929         try:
 930             if not main_topic or not version or not topic:
 931                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 932                                    HTTPStatus.METHOD_NOT_ALLOWED)
 933             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 934                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 935                                    HTTPStatus.METHOD_NOT_ALLOWED)
 936             if version != 'v1':
 937                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 938
 939             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 940                 method = kwargs.pop("METHOD")
 941             else:
 942                 method = cherrypy.request.method
 943
 944             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 945             query_string_operations = self._extract_query_string_operations(kwargs, method)
 946             if main_topic == "admin" and topic == "tokens":
 947                 return self.token(method, _id, kwargs)
 948             token_info = self.authenticator.authorize(role_permission, query_string_operations, _id)
 949             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 950             indata = self._format_in(kwargs)
 951             engine_topic = topic
 952             if topic == "subscriptions":
 953                 engine_topic = main_topic + "_" + topic
 954             if item and topic != "pm_jobs":
 955                 engine_topic = item
 956
 957             if main_topic == "nsd":
 958                 engine_topic = "nsds"
 959             elif main_topic == "vnfpkgm":
 960                 engine_topic = "vnfds"
 961             elif main_topic == "nslcm":
 962                 engine_topic = "nsrs"
 963                 if topic == "ns_lcm_op_occs":
 964                     engine_topic = "nslcmops"
 965                 if topic == "vnfrs" or topic == "vnf_instances":
 966                     engine_topic = "vnfrs"
 967             elif main_topic == "nst":
 968                 engine_topic = "nsts"
 969             elif main_topic == "nsilcm":
 970                 engine_topic = "nsis"
 971                 if topic == "nsi_lcm_op_occs":
 972                     engine_topic = "nsilcmops"
 973             elif main_topic == "pdu":
 974                 engine_topic = "pdus"
 975             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
 976                 engine_topic = "vim_accounts"
 977
 978             if method == "GET":
 979                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
 980                     if item in ("vnfd", "nsd", "nst"):
 981                         path = "$DESCRIPTOR"
 982                     elif args:
 983                         path = args
 984                     elif item == "artifacts":
 985                         path = ()
 986                     else:
 987                         path = None
 988                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
 989                                                          cherrypy.request.headers.get("Accept"))
 990                     outdata = file
 991                 elif not _id:
 992                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
 993                 else:
 994                     if item == "reports":
 995                         # TODO check that project_id (_id in this context) has permissions
 996                         _id = args[0]
 997                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
 998             elif method == "POST":
 999                 cherrypy.response.status = HTTPStatus.CREATED.value
1000                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
1001                     _id = cherrypy.request.headers.get("Transaction-Id")
1002                     if not _id:
1003                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1004                                                       cherrypy.request.headers)
1005                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1006                                                            cherrypy.request.headers)
1007                     if completed:
1008                         self._set_location_header(main_topic, version, topic, _id)
1009                     else:
1010                         cherrypy.response.headers["Transaction-Id"] = _id
1011                     outdata = {"id": _id}
1012                 elif topic == "ns_instances_content":
1013                     # creates NSR
1014                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1015                     # creates nslcmop
1016                     indata["lcmOperationType"] = "instantiate"
1017                     indata["nsInstanceId"] = _id
1018                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1019                     self._set_location_header(main_topic, version, topic, _id)
1020                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1021                 elif topic == "ns_instances" and item:
1022                     indata["lcmOperationType"] = item
1023                     indata["nsInstanceId"] = _id
1024                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1025                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1026                     outdata = {"id": _id}
1027                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1028                 elif topic == "netslice_instances_content":
1029                     # creates NetSlice_Instance_record (NSIR)
1030                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1031                     self._set_location_header(main_topic, version, topic, _id)
1032                     indata["lcmOperationType"] = "instantiate"
1033                     indata["netsliceInstanceId"] = _id
1034                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1035                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1036
1037                 elif topic == "netslice_instances" and item:
1038                     indata["lcmOperationType"] = item
1039                     indata["netsliceInstanceId"] = _id
1040                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1041                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1042                     outdata = {"id": _id}
1043                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1044                 else:
1045                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1046                                                       cherrypy.request.headers)
1047                     self._set_location_header(main_topic, version, topic, _id)
1048                     outdata = {"id": _id}
1049                     if op_id:
1050                         outdata["op_id"] = op_id
1051                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1052                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1053
1054             elif method == "DELETE":
1055                 if not _id:
1056                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1057                     cherrypy.response.status = HTTPStatus.OK.value
1058                 else:  # len(args) > 1
1059                     delete_in_process = False
1060                     if topic == "ns_instances_content" and not engine_session["force"]:
1061                         nslcmop_desc = {
1062                             "lcmOperationType": "terminate",
1063                             "nsInstanceId": _id,
1064                             "autoremove": True
1065                         }
1066                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, None)
1067                         if opp_id:
1068                             delete_in_process = True
1069                             outdata = {"_id": opp_id}
1070                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1071                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1072                         nsilcmop_desc = {
1073                             "lcmOperationType": "terminate",
1074                             "netsliceInstanceId": _id,
1075                             "autoremove": True
1076                         }
1077                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1078                         if opp_id:
1079                             delete_in_process = True
1080                             outdata = {"_id": opp_id}
1081                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1082                     if not delete_in_process:
1083                         self.engine.del_item(engine_session, engine_topic, _id)
1084                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1085                 if engine_topic in ("vim_accounts", "wim_accounts", "sdns"):
1086                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1087
1088             elif method in ("PUT", "PATCH"):
1089                 op_id = None
1090                 if not indata and not kwargs and not engine_session.get("set_project"):
1091                     raise NbiException("Nothing to update. Provide payload and/or query string",
1092                                        HTTPStatus.BAD_REQUEST)
1093                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1094                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1095                                                            cherrypy.request.headers)
1096                     if not completed:
1097                         cherrypy.response.headers["Transaction-Id"] = id
1098                 else:
1099                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1100
1101                 if op_id:
1102                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1103                     outdata = {"op_id": op_id}
1104                 else:
1105                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1106                     outdata = None
1107             else:
1108                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1109
1110             # if Role information changes, it is needed to reload the information of roles
1111             if topic == "roles" and method != "GET":
1112                 self.authenticator.load_operation_to_allowed_roles()
1113             return self._format_out(outdata, token_info, _format)
1114         except Exception as e:
1115             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1116                               ValidationError, AuthconnException)):
1117                 http_code_value = cherrypy.response.status = e.http_code.value
1118                 http_code_name = e.http_code.name
1119                 cherrypy.log("Exception {}".format(e))
1120             else:
1121                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1122                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1123                 http_code_name = HTTPStatus.BAD_REQUEST.name
1124             if hasattr(outdata, "close"):  # is an open file
1125                 outdata.close()
1126             error_text = str(e)
1127             rollback.reverse()
1128             for rollback_item in rollback:
1129                 try:
1130                     if rollback_item.get("operation") == "set":
1131                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1132                                                rollback_item["content"], fail_on_empty=False)
1133                     else:
1134                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1135                                                fail_on_empty=False)
1136                 except Exception as e2:
1137                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1138                     cherrypy.log(rollback_error_text)
1139                     error_text += ". " + rollback_error_text
1140             # if isinstance(e, MsgException):
1141             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1142             #         engine_topic[:-1], method, error_text)
1143             problem_details = {
1144                 "code": http_code_name,
1145                 "status": http_code_value,
1146                 "detail": error_text,
1147             }
1148             return self._format_out(problem_details, token_info)
1149             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1150         finally:
1151             if token_info:
1152                 self._format_login(token_info)
1153                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1154                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1155                         if outdata.get(logging_id):
1156                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1157
1158
1159 def _start_service():
1160     """
1161     Callback function called when cherrypy.engine starts
1162     Override configuration with env variables
1163     Set database, storage, message configuration
1164     Init database with admin/admin user password
1165     """
1166     global nbi_server
1167     global subscription_thread
1168     cherrypy.log.error("Starting osm_nbi")
1169     # update general cherrypy configuration
1170     update_dict = {}
1171
1172     engine_config = cherrypy.tree.apps['/osm'].config
1173     for k, v in environ.items():
1174         if not k.startswith("OSMNBI_"):
1175             continue
1176         k1, _, k2 = k[7:].lower().partition("_")
1177         if not k2:
1178             continue
1179         try:
1180             # update static configuration
1181             if k == 'OSMNBI_STATIC_DIR':
1182                 engine_config["/static"]['tools.staticdir.dir'] = v
1183                 engine_config["/static"]['tools.staticdir.on'] = True
1184             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1185                 update_dict['server.socket_port'] = int(v)
1186             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1187                 update_dict['server.socket_host'] = v
1188             elif k1 in ("server", "test", "auth", "log"):
1189                 update_dict[k1 + '.' + k2] = v
1190             elif k1 in ("message", "database", "storage", "authentication"):
1191                 # k2 = k2.replace('_', '.')
1192                 if k2 in ("port", "db_port"):
1193                     engine_config[k1][k2] = int(v)
1194                 else:
1195                     engine_config[k1][k2] = v
1196
1197         except ValueError as e:
1198             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1199         except Exception as e:
1200             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1201
1202     if update_dict:
1203         cherrypy.config.update(update_dict)
1204         engine_config["global"].update(update_dict)
1205
1206     # logging cherrypy
1207     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1208     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1209     logger_server = logging.getLogger("cherrypy.error")
1210     logger_access = logging.getLogger("cherrypy.access")
1211     logger_cherry = logging.getLogger("cherrypy")
1212     logger_nbi = logging.getLogger("nbi")
1213
1214     if "log.file" in engine_config["global"]:
1215         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1216                                                             maxBytes=100e6, backupCount=9, delay=0)
1217         file_handler.setFormatter(log_formatter_simple)
1218         logger_cherry.addHandler(file_handler)
1219         logger_nbi.addHandler(file_handler)
1220     # log always to standard output
1221     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1222                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1223                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1224                             }.items():
1225         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1226         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1227         str_handler = logging.StreamHandler()
1228         str_handler.setFormatter(log_formatter_cherry)
1229         logger.addHandler(str_handler)
1230
1231     if engine_config["global"].get("log.level"):
1232         logger_cherry.setLevel(engine_config["global"]["log.level"])
1233         logger_nbi.setLevel(engine_config["global"]["log.level"])
1234
1235     # logging other modules
1236     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1237         engine_config[k1]["logger_name"] = logname
1238         logger_module = logging.getLogger(logname)
1239         if "logfile" in engine_config[k1]:
1240             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1241                                                                 maxBytes=100e6, backupCount=9, delay=0)
1242             file_handler.setFormatter(log_formatter_simple)
1243             logger_module.addHandler(file_handler)
1244         if "loglevel" in engine_config[k1]:
1245             logger_module.setLevel(engine_config[k1]["loglevel"])
1246     # TODO add more entries, e.g.: storage
1247     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1248     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1249     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1250     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1251
1252     # start subscriptions thread:
1253     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1254     subscription_thread.start()
1255     # Do not capture except SubscriptionException
1256
1257     # load and print version. Ignore possible errors, e.g. file not found
1258     try:
1259         backend = engine_config["authentication"]["backend"]
1260         nbi_version
1261         cherrypy.log.error("Starting OSM NBI Version '{}' with '{}' authentication backend"
1262                            .format(nbi_version + " " + nbi_version_date, backend))
1263     except Exception:
1264         pass
1265
1266
1267 def _stop_service():
1268     """
1269     Callback function called when cherrypy.engine stops
1270     TODO: Ending database connections.
1271     """
1272     global subscription_thread
1273     if subscription_thread:
1274         subscription_thread.terminate()
1275     subscription_thread = None
1276     cherrypy.tree.apps['/osm'].root.engine.stop()
1277     cherrypy.log.error("Stopping osm_nbi")
1278
1279
1280 def nbi(config_file):
1281     global nbi_server
1282     # conf = {
1283     #     '/': {
1284     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1285     #         'tools.sessions.on': True,
1286     #         'tools.response_headers.on': True,
1287     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1288     #     }
1289     # }
1290     # cherrypy.Server.ssl_module = 'builtin'
1291     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1292     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1293     # cherrypy.Server.thread_pool = 10
1294     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1295
1296     # cherrypy.config.update({'tools.auth_basic.on': True,
1297     #    'tools.auth_basic.realm': 'localhost',
1298     #    'tools.auth_basic.checkpassword': validate_password})
1299     nbi_server = Server()
1300     cherrypy.engine.subscribe('start', _start_service)
1301     cherrypy.engine.subscribe('stop', _stop_service)
1302     cherrypy.quickstart(nbi_server, '/osm', config_file)
1303
1304
1305 def usage():
1306     print("""Usage: {} [options]
1307         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1308         -h|--help: shows this help
1309         """.format(sys.argv[0]))
1310     # --log-socket-host HOST: send logs to this host")
1311     # --log-socket-port PORT: send logs using this port (default: 9022)")
1312
1313
1314 if __name__ == '__main__':
1315     try:
1316         # load parameters and configuration
1317         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1318         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1319         config_file = None
1320         for o, a in opts:
1321             if o in ("-h", "--help"):
1322                 usage()
1323                 sys.exit()
1324             elif o in ("-c", "--config"):
1325                 config_file = a
1326             # elif o == "--log-socket-port":
1327             #     log_socket_port = a
1328             # elif o == "--log-socket-host":
1329             #     log_socket_host = a
1330             # elif o == "--log-file":
1331             #     log_file = a
1332             else:
1333                 assert False, "Unhandled option"
1334         if config_file:
1335             if not path.isfile(config_file):
1336                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1337                 exit(1)
1338         else:
1339             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1340                 if path.isfile(config_file):
1341                     break
1342             else:
1343                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1344                 exit(1)
1345         nbi(config_file)
1346     except getopt.GetoptError as e:
1347         print(str(e), file=sys.stderr)
1348         # usage()
1349         exit(1)