osm_nbi/nbi.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import cherrypy
  18 import time
  19 import json
  20 import yaml
  21 import html_out as html
  22 import logging
  23 import logging.handlers
  24 import getopt
  25 import sys
  26
  27 from authconn import AuthException, AuthconnException
  28 from auth import Authenticator
  29 from engine import Engine, EngineException
  30 from subscriptions import SubscriptionThread
  31 from validation import ValidationError
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34 from osm_common.msgbase import MsgException
  35 from http import HTTPStatus
  36 from codecs import getreader
  37 from os import environ, path
  38
  39 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  40
  41 __version__ = "0.1.3"
  42 version_date = "Jan 2019"
  43 database_version = '1.2'
  44 auth_database_version = '1.0'
  45 nbi_server = None           # instance of Server class
  46 subscription_thread = None  # instance of SubscriptionThread class
  47
  48
  49 """
  50 North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
  51 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
  52         /nsd/v1
  53             /ns_descriptors_content                             O       O
  54                 /<nsdInfoId>                                    O       O       O       O
  55             /ns_descriptors                                     O5      O5
  56                 /<nsdInfoId>                                    O5                      O5      5
  57                     /nsd_content                                O5              O5
  58                     /nsd                                        O
  59                     /artifacts[/<artifactPath>]                 O
  60             /pnf_descriptors                                    5       5
  61                 /<pnfdInfoId>                                   5                       5       5
  62                     /pnfd_content                               5               5
  63             /subscriptions                                      5       5
  64                 /<subscriptionId>                               5                       X
  65
  66         /vnfpkgm/v1
  67             /vnf_packages_content                               O       O
  68                 /<vnfPkgId>                                     O                       O
  69             /vnf_packages                                       O5      O5
  70                 /<vnfPkgId>                                     O5                      O5      5
  71                     /package_content                            O5               O5
  72                         /upload_from_uri                                X
  73                     /vnfd                                       O5
  74                     /artifacts[/<artifactPath>]                 O5
  75             /subscriptions                                      X       X
  76                 /<subscriptionId>                               X                       X
  77
  78         /nslcm/v1
  79             /ns_instances_content                               O       O
  80                 /<nsInstanceId>                                 O                       O
  81             /ns_instances                                       5       5
  82                 /<nsInstanceId>                                 O5                      O5
  83                     instantiate                                         O5
  84                     terminate                                           O5
  85                     action                                              O
  86                     scale                                               O5
  87                     heal                                                5
  88             /ns_lcm_op_occs                                     5       5
  89                 /<nsLcmOpOccId>                                 5                       5       5
  90                     TO BE COMPLETED                             5               5
  91             /vnf_instances  (also vnfrs for compatibility)      O
  92                 /<vnfInstanceId>                                O
  93             /subscriptions                                      5       5
  94                 /<subscriptionId>                               5                       X
  95
  96         /pdu/v1
  97             /pdu_descriptors                                    O       O
  98                 /<id>                                           O               O       O       O
  99
 100         /admin/v1
 101             /tokens                                             O       O
 102                 /<id>                                           O                       O
 103             /users                                              O       O
 104                 /<id>                                           O               O       O       O
 105             /projects                                           O       O
 106                 /<id>                                           O                       O
 107             /vim_accounts  (also vims for compatibility)        O       O
 108                 /<id>                                           O                       O       O
 109             /wim_accounts                                       O       O
 110                 /<id>                                           O                       O       O
 111             /sdns                                               O       O
 112                 /<id>                                           O                       O       O
 113
 114         /nst/v1                                                 O       O
 115             /netslice_templates_content                         O       O
 116                 /<nstInfoId>                                    O       O       O       O
 117             /netslice_templates                                 O       O
 118                 /<nstInfoId>                                    O                       O       O
 119                     /nst_content                                O               O
 120                     /nst                                        O
 121                     /artifacts[/<artifactPath>]                 O
 122             /subscriptions                                      X       X
 123                 /<subscriptionId>                               X                       X
 124
 125         /nsilcm/v1
 126             /netslice_instances_content                         O       O
 127                 /<SliceInstanceId>                              O                       O
 128             /netslice_instances                                 O       O
 129                 /<SliceInstanceId>                              O                       O
 130                     instantiate                                         O
 131                     terminate                                           O
 132                     action                                              O
 133             /nsi_lcm_op_occs                                    O       O
 134                 /<nsiLcmOpOccId>                                O                       O       O
 135             /subscriptions                                      X       X
 136                 /<subscriptionId>                               X                       X
 137
 138 query string:
 139     Follows SOL005 section 4.3.2 It contains extra METHOD to override http method, FORCE to force.
 140         simpleFilterExpr := <attrName>["."<attrName>]*["."<op>]"="<value>[","<value>]*
 141         filterExpr := <simpleFilterExpr>["&"<simpleFilterExpr>]*
 142         op := "eq" | "neq" (or "ne") | "gt" | "lt" | "gte" | "lte" | "cont" | "ncont"
 143         attrName := string
 144     For filtering inside array, it must select the element of the array, or add ANYINDEX to apply the filtering over any
 145     item of the array, that is, pass if any item of the array pass the filter.
 146     It allows both ne and neq for not equal
 147     TODO: 4.3.3 Attribute selectors
 148         all_fields, fields=x,y,.., exclude_default, exclude_fields=x,y,...
 149         (none)  … same as “exclude_default”
 150         all_fields      … all attributes.
 151         fields=<list>   … all attributes except all complex attributes with minimum cardinality of zero that are not
 152         conditionally mandatory, and that are not provided in <list>.
 153         exclude_fields=<list>   … all attributes except those complex attributes with a minimum cardinality of zero that
 154         are not conditionally mandatory, and that are provided in <list>.
 155         exclude_default … all attributes except those complex attributes with a minimum cardinality of zero that are not
 156         conditionally mandatory, and that are part of the "default exclude set" defined in the present specification for
 157         the particular resource
 158         exclude_default and include=<list>      … all attributes except those complex attributes with a minimum cardinality
 159         of zero that are not conditionally mandatory and that are part of the "default exclude set" defined in the
 160         present specification for the particular resource, but that are not part of <list>
 161     Additionally it admits some administrator values:
 162         FORCE: To force operations skipping dependency checkings
 163         ADMIN: To act as an administrator or a different project
 164         PUBLIC: To get public descriptors or set a descriptor as public
 165         SET_PROJECT: To make a descriptor available for other project
 166
 167 Header field name       Reference       Example Descriptions
 168     Accept      IETF RFC 7231 [19]      application/json        Content-Types that are acceptable for the response.
 169     This header field shall be present if the response is expected to have a non-empty message body.
 170     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the request.
 171     This header field shall be present if the request has a non-empty message body.
 172     Authorization       IETF RFC 7235 [22]      Bearer mF_9.B5f-4.1JqM  The authorization token for the request.
 173     Details are specified in clause 4.5.3.
 174     Range       IETF RFC 7233 [21]      1000-2000       Requested range of bytes from a file
 175 Header field name       Reference       Example Descriptions
 176     Content-Type        IETF RFC 7231 [19]      application/json        The MIME type of the body of the response.
 177     This header field shall be present if the response has a non-empty message body.
 178     Location    IETF RFC 7231 [19]      http://www.example.com/vnflcm/v1/vnf_instances/123      Used in redirection, or when a
 179     new resource has been created.
 180     This header field shall be present if the response status code is 201 or 3xx.
 181     In the present document this header field is also used if the response status code is 202 and a new resource was
 182     created.
 183     WWW-Authenticate    IETF RFC 7235 [22]      Bearer realm="example"  Challenge if the corresponding HTTP request has not
 184     provided authorization, or error details if the corresponding HTTP request has provided an invalid authorization
 185     token.
 186     Accept-Ranges       IETF RFC 7233 [21]      bytes   Used by the Server to signal whether or not it supports ranges for
 187     certain resources.
 188     Content-Range       IETF RFC 7233 [21]      bytes 21010-47021/ 47022        Signals the byte range that is contained in the
 189     response, and the total length of the file.
 190     Retry-After IETF RFC 7231 [19]      Fri, 31 Dec 1999 23:59:59 GMT
 191 """
 192
 193 valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
 194 # ^ Contains possible administrative query string words:
 195 #     ADMIN=True(by default)|Project|Project-list:  See all elements, or elements of a project
 196 #           (not owned by my session project).
 197 #     PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
 198 #     FORCE=True(by default)|False: Force edition/deletion operations
 199 #     SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
 200
 201 valid_url_methods = {
 202     # contains allowed URL and methods, and the role_permission name
 203     "admin": {
 204         "v1": {
 205             "tokens": {"METHODS": ("GET", "POST", "DELETE"),
 206                        "ROLE_PERMISSION": "tokens:",
 207                        "<ID>": {"METHODS": ("GET", "DELETE"),
 208                                 "ROLE_PERMISSION": "tokens:id:"
 209                                 }
 210                        },
 211             "users": {"METHODS": ("GET", "POST"),
 212                       "ROLE_PERMISSION": "users:",
 213                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 214                                "ROLE_PERMISSION": "users:id:"
 215                                }
 216                       },
 217             "projects": {"METHODS": ("GET", "POST"),
 218                          "ROLE_PERMISSION": "projects:",
 219                          "<ID>": {"METHODS": ("GET", "DELETE", "PUT"),
 220                                   "ROLE_PERMISSION": "projects:id:"}
 221                          },
 222             "roles": {"METHODS": ("GET", "POST"),
 223                       "ROLE_PERMISSION": "roles:",
 224                       "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PUT"),
 225                                "ROLE_PERMISSION": "roles:id:"
 226                                }
 227                       },
 228             "vims": {"METHODS": ("GET", "POST"),
 229                      "ROLE_PERMISSION": "vims:",
 230                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 231                               "ROLE_PERMISSION": "vims:id:"
 232                               }
 233                      },
 234             "vim_accounts": {"METHODS": ("GET", "POST"),
 235                              "ROLE_PERMISSION": "vim_accounts:",
 236                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 237                                       "ROLE_PERMISSION": "vim_accounts:id:"
 238                                       }
 239                              },
 240             "wim_accounts": {"METHODS": ("GET", "POST"),
 241                              "ROLE_PERMISSION": "wim_accounts:",
 242                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 243                                       "ROLE_PERMISSION": "wim_accounts:id:"
 244                                       }
 245                              },
 246             "sdns": {"METHODS": ("GET", "POST"),
 247                      "ROLE_PERMISSION": "sdn_controllers:",
 248                      "<ID>": {"METHODS": ("GET", "DELETE", "PATCH", "PUT"),
 249                               "ROLE_PERMISSION": "sdn_controllers:id:"
 250                               }
 251                      },
 252         }
 253     },
 254     "pdu": {
 255         "v1": {
 256             "pdu_descriptors": {"METHODS": ("GET", "POST"),
 257                                 "ROLE_PERMISSION": "pduds:",
 258                                 "<ID>": {"METHODS": ("GET", "POST", "DELETE", "PATCH", "PUT"),
 259                                          "ROLE_PERMISSION": "pduds:id:"
 260                                          }
 261                                 },
 262         }
 263     },
 264     "nsd": {
 265         "v1": {
 266             "ns_descriptors_content": {"METHODS": ("GET", "POST"),
 267                                        "ROLE_PERMISSION": "nsds:",
 268                                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 269                                                 "ROLE_PERMISSION": "nsds:id:"
 270                                                 }
 271                                        },
 272             "ns_descriptors": {"METHODS": ("GET", "POST"),
 273                                "ROLE_PERMISSION": "nsds:",
 274                                "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),
 275                                         "ROLE_PERMISSION": "nsds:id:",
 276                                         "nsd_content": {"METHODS": ("GET", "PUT"),
 277                                                         "ROLE_PERMISSION": "nsds:id:content:",
 278                                                         },
 279                                         "nsd": {"METHODS": ("GET",),  # descriptor inside package
 280                                                 "ROLE_PERMISSION": "nsds:id:content:"
 281                                                 },
 282                                         "artifacts": {"*": {"METHODS": ("GET",),
 283                                                             "ROLE_PERMISSION": "nsds:id:nsd_artifact:"
 284                                                             }
 285                                                       }
 286                                         }
 287                                },
 288             "pnf_descriptors": {"TODO": ("GET", "POST"),
 289                                 "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
 290                                          "pnfd_content": {"TODO": ("GET", "PUT")}
 291                                          }
 292                                 },
 293             "subscriptions": {"TODO": ("GET", "POST"),
 294                               "<ID>": {"TODO": ("GET", "DELETE")}
 295                               },
 296         }
 297     },
 298     "vnfpkgm": {
 299         "v1": {
 300             "vnf_packages_content": {"METHODS": ("GET", "POST"),
 301                                      "ROLE_PERMISSION": "vnfds:",
 302                                      "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 303                                               "ROLE_PERMISSION": "vnfds:id:"}
 304                                      },
 305             "vnf_packages": {"METHODS": ("GET", "POST"),
 306                              "ROLE_PERMISSION": "vnfds:",
 307                              "<ID>": {"METHODS": ("GET", "DELETE", "PATCH"),  # GET: vnfPkgInfo
 308                                       "ROLE_PERMISSION": "vnfds:id:",
 309                                       "package_content": {"METHODS": ("GET", "PUT"),  # package
 310                                                           "ROLE_PERMISSION": "vnfds:id:",
 311                                                           "upload_from_uri": {"METHODS": (),
 312                                                                               "TODO": ("POST", ),
 313                                                                               "ROLE_PERMISSION": "vnfds:id:upload:"
 314                                                                               }
 315                                                           },
 316                                       "vnfd": {"METHODS": ("GET", ),  # descriptor inside package
 317                                                "ROLE_PERMISSION": "vnfds:id:content:"
 318                                                },
 319                                       "artifacts": {"*": {"METHODS": ("GET", ),
 320                                                           "ROLE_PERMISSION": "vnfds:id:vnfd_artifact:"
 321                                                           }
 322                                                     }
 323                                       }
 324                              },
 325             "subscriptions": {"TODO": ("GET", "POST"),
 326                               "<ID>": {"TODO": ("GET", "DELETE")}
 327                               },
 328         }
 329     },
 330     "nslcm": {
 331         "v1": {
 332             "ns_instances_content": {"METHODS": ("GET", "POST"),
 333                                      "ROLE_PERMISSION": "ns_instances:",
 334                                      "<ID>": {"METHODS": ("GET", "DELETE"),
 335                                               "ROLE_PERMISSION": "ns_instances:id:"
 336                                               }
 337                                      },
 338             "ns_instances": {"METHODS": ("GET", "POST"),
 339                              "ROLE_PERMISSION": "ns_instances:",
 340                              "<ID>": {"METHODS": ("GET", "DELETE"),
 341                                       "ROLE_PERMISSION": "ns_instances:id:",
 342                                       "scale": {"METHODS": ("POST",),
 343                                                 "ROLE_PERMISSION": "ns_instances:id:scale:"
 344                                                 },
 345                                       "terminate": {"METHODS": ("POST",),
 346                                                     "ROLE_PERMISSION": "ns_instances:id:terminate:"
 347                                                     },
 348                                       "instantiate": {"METHODS": ("POST",),
 349                                                       "ROLE_PERMISSION": "ns_instances:id:instantiate:"
 350                                                       },
 351                                       "action": {"METHODS": ("POST",),
 352                                                  "ROLE_PERMISSION": "ns_instances:id:action:"
 353                                                  },
 354                                       }
 355                              },
 356             "ns_lcm_op_occs": {"METHODS": ("GET",),
 357                                "ROLE_PERMISSION": "ns_instances:opps:",
 358                                "<ID>": {"METHODS": ("GET",),
 359                                         "ROLE_PERMISSION": "ns_instances:opps:id:"
 360                                         },
 361                                },
 362             "vnfrs": {"METHODS": ("GET",),
 363                       "ROLE_PERMISSION": "vnf_instances:",
 364                       "<ID>": {"METHODS": ("GET",),
 365                                "ROLE_PERMISSION": "vnf_instances:id:"
 366                                }
 367                       },
 368             "vnf_instances": {"METHODS": ("GET",),
 369                               "ROLE_PERMISSION": "vnf_instances:",
 370                               "<ID>": {"METHODS": ("GET",),
 371                                        "ROLE_PERMISSION": "vnf_instances:id:"
 372                                        }
 373                               },
 374         }
 375     },
 376     "nst": {
 377         "v1": {
 378             "netslice_templates_content": {"METHODS": ("GET", "POST"),
 379                                            "ROLE_PERMISSION": "slice_templates:",
 380                                            "<ID>": {"METHODS": ("GET", "PUT", "DELETE"),
 381                                                     "ROLE_PERMISSION": "slice_templates:id:", }
 382                                            },
 383             "netslice_templates": {"METHODS": ("GET", "POST"),
 384                                    "ROLE_PERMISSION": "slice_templates:",
 385                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 386                                             "TODO": ("PATCH",),
 387                                             "ROLE_PERMISSION": "slice_templates:id:",
 388                                             "nst_content": {"METHODS": ("GET", "PUT"),
 389                                                             "ROLE_PERMISSION": "slice_templates:id:content:"
 390                                                             },
 391                                             "nst": {"METHODS": ("GET",),  # descriptor inside package
 392                                                     "ROLE_PERMISSION": "slice_templates:id:content:"
 393                                                     },
 394                                             "artifacts": {"*": {"METHODS": ("GET",),
 395                                                                 "ROLE_PERMISSION": "slice_templates:id:content:"
 396                                                                 }
 397                                                           }
 398                                             }
 399                                    },
 400             "subscriptions": {"TODO": ("GET", "POST"),
 401                               "<ID>": {"TODO": ("GET", "DELETE")}
 402                               },
 403         }
 404     },
 405     "nsilcm": {
 406         "v1": {
 407             "netslice_instances_content": {"METHODS": ("GET", "POST"),
 408                                            "ROLE_PERMISSION": "slice_instances:",
 409                                            "<ID>": {"METHODS": ("GET", "DELETE"),
 410                                                     "ROLE_PERMISSION": "slice_instances:id:"
 411                                                     }
 412                                            },
 413             "netslice_instances": {"METHODS": ("GET", "POST"),
 414                                    "ROLE_PERMISSION": "slice_instances:",
 415                                    "<ID>": {"METHODS": ("GET", "DELETE"),
 416                                             "ROLE_PERMISSION": "slice_instances:id:",
 417                                             "terminate": {"METHODS": ("POST",),
 418                                                           "ROLE_PERMISSION": "slice_instances:id:terminate:"
 419                                                           },
 420                                             "instantiate": {"METHODS": ("POST",),
 421                                                             "ROLE_PERMISSION": "slice_instances:id:instantiate:"
 422                                                             },
 423                                             "action": {"METHODS": ("POST",),
 424                                                        "ROLE_PERMISSION": "slice_instances:id:action:"
 425                                                        },
 426                                             }
 427                                    },
 428             "nsi_lcm_op_occs": {"METHODS": ("GET",),
 429                                 "ROLE_PERMISSION": "slice_instances:opps:",
 430                                 "<ID>": {"METHODS": ("GET",),
 431                                          "ROLE_PERMISSION": "slice_instances:opps:id:",
 432                                          },
 433                                 },
 434         }
 435     },
 436     "nspm": {
 437         "v1": {
 438             "pm_jobs": {
 439                 "<ID>": {
 440                     "reports": {
 441                         "<ID>": {"METHODS": ("GET",),
 442                                  "ROLE_PERMISSION": "reports:id:",
 443                                  }
 444                     }
 445                 },
 446             },
 447         },
 448     },
 449 }
 450
 451
 452 class NbiException(Exception):
 453
 454     def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
 455         Exception.__init__(self, message)
 456         self.http_code = http_code
 457
 458
 459 class Server(object):
 460     instance = 0
 461     # to decode bytes to str
 462     reader = getreader("utf-8")
 463
 464     def __init__(self):
 465         self.instance += 1
 466         self.engine = Engine()
 467         self.authenticator = Authenticator(valid_url_methods, valid_query_string)
 468
 469     def _format_in(self, kwargs):
 470         try:
 471             indata = None
 472             if cherrypy.request.body.length:
 473                 error_text = "Invalid input format "
 474
 475                 if "Content-Type" in cherrypy.request.headers:
 476                     if "application/json" in cherrypy.request.headers["Content-Type"]:
 477                         error_text = "Invalid json format "
 478                         indata = json.load(self.reader(cherrypy.request.body))
 479                         cherrypy.request.headers.pop("Content-File-MD5", None)
 480                     elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
 481                         error_text = "Invalid yaml format "
 482                         indata = yaml.load(cherrypy.request.body)
 483                         cherrypy.request.headers.pop("Content-File-MD5", None)
 484                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
 485                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
 486                          "application/zip" in cherrypy.request.headers["Content-Type"] or \
 487                          "text/plain" in cherrypy.request.headers["Content-Type"]:
 488                         indata = cherrypy.request.body  # .read()
 489                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
 490                         if "descriptor_file" in kwargs:
 491                             filecontent = kwargs.pop("descriptor_file")
 492                             if not filecontent.file:
 493                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
 494                             indata = filecontent.file  # .read()
 495                             if filecontent.content_type.value:
 496                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
 497                     else:
 498                         # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
 499                         #                          "Only 'Content-Type' of type 'application/json' or
 500                         # 'application/yaml' for input format are available")
 501                         error_text = "Invalid yaml format "
 502                         indata = yaml.load(cherrypy.request.body)
 503                         cherrypy.request.headers.pop("Content-File-MD5", None)
 504                 else:
 505                     error_text = "Invalid yaml format "
 506                     indata = yaml.load(cherrypy.request.body)
 507                     cherrypy.request.headers.pop("Content-File-MD5", None)
 508             if not indata:
 509                 indata = {}
 510
 511             format_yaml = False
 512             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
 513                 format_yaml = True
 514
 515             for k, v in kwargs.items():
 516                 if isinstance(v, str):
 517                     if v == "":
 518                         kwargs[k] = None
 519                     elif format_yaml:
 520                         try:
 521                             kwargs[k] = yaml.load(v)
 522                         except Exception:
 523                             pass
 524                     elif k.endswith(".gt") or k.endswith(".lt") or k.endswith(".gte") or k.endswith(".lte"):
 525                         try:
 526                             kwargs[k] = int(v)
 527                         except Exception:
 528                             try:
 529                                 kwargs[k] = float(v)
 530                             except Exception:
 531                                 pass
 532                     elif v.find(",") > 0:
 533                         kwargs[k] = v.split(",")
 534                 elif isinstance(v, (list, tuple)):
 535                     for index in range(0, len(v)):
 536                         if v[index] == "":
 537                             v[index] = None
 538                         elif format_yaml:
 539                             try:
 540                                 v[index] = yaml.load(v[index])
 541                             except Exception:
 542                                 pass
 543
 544             return indata
 545         except (ValueError, yaml.YAMLError) as exc:
 546             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 547         except KeyError as exc:
 548             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 549         except Exception as exc:
 550             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
 551
 552     @staticmethod
 553     def _format_out(data, token_info=None, _format=None):
 554         """
 555         return string of dictionary data according to requested json, yaml, xml. By default json
 556         :param data: response to be sent. Can be a dict, text or file
 557         :param token_info: Contains among other username and project
 558         :param _format: The format to be set as Content-Type ir data is a file
 559         :return: None
 560         """
 561         accept = cherrypy.request.headers.get("Accept")
 562         if data is None:
 563             if accept and "text/html" in accept:
 564                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 565             # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
 566             return
 567         elif hasattr(data, "read"):  # file object
 568             if _format:
 569                 cherrypy.response.headers["Content-Type"] = _format
 570             elif "b" in data.mode:  # binariy asssumig zip
 571                 cherrypy.response.headers["Content-Type"] = 'application/zip'
 572             else:
 573                 cherrypy.response.headers["Content-Type"] = 'text/plain'
 574             # TODO check that cherrypy close file. If not implement pending things to close  per thread next
 575             return data
 576         if accept:
 577             if "application/json" in accept:
 578                 cherrypy.response.headers["Content-Type"] = 'application/json; charset=utf-8'
 579                 a = json.dumps(data, indent=4) + "\n"
 580                 return a.encode("utf8")
 581             elif "text/html" in accept:
 582                 return html.format(data, cherrypy.request, cherrypy.response, token_info)
 583
 584             elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
 585                 pass
 586             # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
 587             elif cherrypy.response.status >= 400:
 588                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
 589                                          "Only 'Accept' of type 'application/json' or 'application/yaml' "
 590                                          "for output format are available")
 591         cherrypy.response.headers["Content-Type"] = 'application/yaml'
 592         return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False,
 593                               encoding='utf-8', allow_unicode=True)  # , canonical=True, default_style='"'
 594
 595     @cherrypy.expose
 596     def index(self, *args, **kwargs):
 597         token_info = None
 598         try:
 599             if cherrypy.request.method == "GET":
 600                 token_info = self.authenticator.authorize()
 601                 outdata = token_info   # Home page
 602             else:
 603                 raise cherrypy.HTTPError(HTTPStatus.METHOD_NOT_ALLOWED.value,
 604                                          "Method {} not allowed for tokens".format(cherrypy.request.method))
 605
 606             return self._format_out(outdata, token_info)
 607
 608         except (EngineException, AuthException) as e:
 609             # cherrypy.log("index Exception {}".format(e))
 610             cherrypy.response.status = e.http_code.value
 611             return self._format_out("Welcome to OSM!", token_info)
 612
 613     @cherrypy.expose
 614     def version(self, *args, **kwargs):
 615         # TODO consider to remove and provide version using the static version file
 616         global __version__, version_date
 617         try:
 618             if cherrypy.request.method != "GET":
 619                 raise NbiException("Only method GET is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
 620             elif args or kwargs:
 621                 raise NbiException("Invalid URL or query string for version", HTTPStatus.METHOD_NOT_ALLOWED)
 622             return __version__ + " " + version_date
 623         except NbiException as e:
 624             cherrypy.response.status = e.http_code.value
 625             problem_details = {
 626                 "code": e.http_code.name,
 627                 "status": e.http_code.value,
 628                 "detail": str(e),
 629             }
 630             return self._format_out(problem_details, None)
 631
 632     @staticmethod
 633     def _format_login(token_info):
 634         """
 635         Changes cherrypy.request.login to include username/project_name;session so that cherrypy access log will
 636         log this information
 637         :param token_info: Dictionary with token content
 638         :return: None
 639         """
 640         cherrypy.request.login = token_info.get("username", "-")
 641         if token_info.get("project_name"):
 642             cherrypy.request.login += "/" + token_info["project_name"]
 643         if token_info.get("id"):
 644             cherrypy.request.login += ";session=" + token_info["id"][0:12]
 645
 646     @cherrypy.expose
 647     def token(self, method, token_id=None, kwargs=None):
 648         token_info = None
 649         # self.engine.load_dbase(cherrypy.request.app.config)
 650         indata = self._format_in(kwargs)
 651         if not isinstance(indata, dict):
 652             raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
 653
 654         if method == "GET":
 655             token_info = self.authenticator.authorize()
 656             # for logging
 657             self._format_login(token_info)
 658             if token_id:
 659                 outdata = self.authenticator.get_token(token_info, token_id)
 660             else:
 661                 outdata = self.authenticator.get_token_list(token_info)
 662         elif method == "POST":
 663             try:
 664                 token_info = self.authenticator.authorize()
 665             except Exception:
 666                 token_info = None
 667             if kwargs:
 668                 indata.update(kwargs)
 669             # This is needed to log the user when authentication fails
 670             cherrypy.request.login = "{}".format(indata.get("username", "-"))
 671             outdata = token_info = self.authenticator.new_token(token_info, indata, cherrypy.request.remote)
 672             cherrypy.session['Authorization'] = outdata["_id"]
 673             self._set_location_header("admin", "v1", "tokens", outdata["_id"])
 674             # for logging
 675             self._format_login(token_info)
 676
 677             # cherrypy.response.cookie["Authorization"] = outdata["id"]
 678             # cherrypy.response.cookie["Authorization"]['expires'] = 3600
 679         elif method == "DELETE":
 680             if not token_id and "id" in kwargs:
 681                 token_id = kwargs["id"]
 682             elif not token_id:
 683                 token_info = self.authenticator.authorize()
 684                 # for logging
 685                 self._format_login(token_info)
 686                 token_id = token_info["_id"]
 687             outdata = self.authenticator.del_token(token_id)
 688             token_info = None
 689             cherrypy.session['Authorization'] = "logout"
 690             # cherrypy.response.cookie["Authorization"] = token_id
 691             # cherrypy.response.cookie["Authorization"]['expires'] = 0
 692         else:
 693             raise NbiException("Method {} not allowed for token".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 694         return self._format_out(outdata, token_info)
 695
 696     @cherrypy.expose
 697     def test(self, *args, **kwargs):
 698         thread_info = None
 699         if args and args[0] == "help":
 700             return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
 701                    "sleep/<time>\nmessage/topic\n</pre></html>"
 702
 703         elif args and args[0] == "init":
 704             try:
 705                 # self.engine.load_dbase(cherrypy.request.app.config)
 706                 self.engine.create_admin()
 707                 return "Done. User 'admin', password 'admin' created"
 708             except Exception:
 709                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 710                 return self._format_out("Database already initialized")
 711         elif args and args[0] == "file":
 712             return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
 713                                                   "text/plain", "attachment")
 714         elif args and args[0] == "file2":
 715             f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
 716             f = open(f_path, "r")
 717             cherrypy.response.headers["Content-type"] = "text/plain"
 718             return f
 719
 720         elif len(args) == 2 and args[0] == "db-clear":
 721             deleted_info = self.engine.db.del_list(args[1], kwargs)
 722             return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
 723         elif len(args) and args[0] == "fs-clear":
 724             if len(args) >= 2:
 725                 folders = (args[1],)
 726             else:
 727                 folders = self.engine.fs.dir_ls(".")
 728             for folder in folders:
 729                 self.engine.fs.file_delete(folder)
 730             return ",".join(folders) + " folders deleted\n"
 731         elif args and args[0] == "login":
 732             if not cherrypy.request.headers.get("Authorization"):
 733                 cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"'
 734                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 735         elif args and args[0] == "login2":
 736             if not cherrypy.request.headers.get("Authorization"):
 737                 cherrypy.response.headers["WWW-Authenticate"] = 'Bearer realm="Access to OSM site"'
 738                 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
 739         elif args and args[0] == "sleep":
 740             sleep_time = 5
 741             try:
 742                 sleep_time = int(args[1])
 743             except Exception:
 744                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
 745                 return self._format_out("Database already initialized")
 746             thread_info = cherrypy.thread_data
 747             print(thread_info)
 748             time.sleep(sleep_time)
 749             # thread_info
 750         elif len(args) >= 2 and args[0] == "message":
 751             main_topic = args[1]
 752             return_text = "<html><pre>{} ->\n".format(main_topic)
 753             try:
 754                 if cherrypy.request.method == 'POST':
 755                     to_send = yaml.load(cherrypy.request.body)
 756                     for k, v in to_send.items():
 757                         self.engine.msg.write(main_topic, k, v)
 758                         return_text += "  {}: {}\n".format(k, v)
 759                 elif cherrypy.request.method == 'GET':
 760                     for k, v in kwargs.items():
 761                         self.engine.msg.write(main_topic, k, yaml.load(v))
 762                         return_text += "  {}: {}\n".format(k, yaml.load(v))
 763             except Exception as e:
 764                 return_text += "Error: " + str(e)
 765             return_text += "</pre></html>\n"
 766             return return_text
 767
 768         return_text = (
 769             "<html><pre>\nheaders:\n  args: {}\n".format(args) +
 770             "  kwargs: {}\n".format(kwargs) +
 771             "  headers: {}\n".format(cherrypy.request.headers) +
 772             "  path_info: {}\n".format(cherrypy.request.path_info) +
 773             "  query_string: {}\n".format(cherrypy.request.query_string) +
 774             "  session: {}\n".format(cherrypy.session) +
 775             "  cookie: {}\n".format(cherrypy.request.cookie) +
 776             "  method: {}\n".format(cherrypy.request.method) +
 777             "  session: {}\n".format(cherrypy.session.get('fieldname')) +
 778             "  body:\n")
 779         return_text += "    length: {}\n".format(cherrypy.request.body.length)
 780         if cherrypy.request.body.length:
 781             return_text += "    content: {}\n".format(
 782                 str(cherrypy.request.body.read(int(cherrypy.request.headers.get('Content-Length', 0)))))
 783         if thread_info:
 784             return_text += "thread: {}\n".format(thread_info)
 785         return_text += "</pre></html>"
 786         return return_text
 787
 788     @staticmethod
 789     def _check_valid_url_method(method, *args):
 790         if len(args) < 3:
 791             raise NbiException("URL must contain at least 'main_topic/version/topic'", HTTPStatus.METHOD_NOT_ALLOWED)
 792
 793         reference = valid_url_methods
 794         for arg in args:
 795             if arg is None:
 796                 break
 797             if not isinstance(reference, dict):
 798                 raise NbiException("URL contains unexpected extra items '{}'".format(arg),
 799                                    HTTPStatus.METHOD_NOT_ALLOWED)
 800
 801             if arg in reference:
 802                 reference = reference[arg]
 803             elif "<ID>" in reference:
 804                 reference = reference["<ID>"]
 805             elif "*" in reference:
 806                 reference = reference["*"]
 807                 break
 808             else:
 809                 raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
 810         if "TODO" in reference and method in reference["TODO"]:
 811             raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
 812         elif "METHODS" in reference and method not in reference["METHODS"]:
 813             raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
 814         return reference["ROLE_PERMISSION"] + method.lower()
 815
 816     @staticmethod
 817     def _set_location_header(main_topic, version, topic, id):
 818         """
 819         Insert response header Location with the URL of created item base on URL params
 820         :param main_topic:
 821         :param version:
 822         :param topic:
 823         :param id:
 824         :return: None
 825         """
 826         # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
 827         cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(main_topic, version, topic, id)
 828         return
 829
 830     @staticmethod
 831     def _extract_query_string_operations(kwargs, method):
 832         """
 833
 834         :param kwargs:
 835         :return:
 836         """
 837         query_string_operations = []
 838         if kwargs:
 839             for qs in ("FORCE", "PUBLIC", "ADMIN", "SET_PROJECT"):
 840                 if qs in kwargs and kwargs[qs].lower() != "false":
 841                     query_string_operations.append(qs.lower() + ":" + method.lower())
 842         return query_string_operations
 843
 844     @staticmethod
 845     def _manage_admin_query(token_info, kwargs, method, _id):
 846         """
 847         Processes the administrator query inputs (if any) of FORCE, ADMIN, PUBLIC, SET_PROJECT
 848         Check that users has rights to use them and returs the admin_query
 849         :param token_info: token_info rights obtained by token
 850         :param kwargs: query string input.
 851         :param method: http method: GET, POSST, PUT, ...
 852         :param _id:
 853         :return: admin_query dictionary with keys:
 854             public: True, False or None
 855             force: True or False
 856             project_id: tuple with projects used for accessing an element
 857             set_project: tuple with projects that a created element will belong to
 858             method: show, list, delete, write
 859         """
 860         admin_query = {"force": False, "project_id": (token_info["project_id"], ), "username": token_info["username"],
 861                        "admin": token_info["admin"], "public": None}
 862         if kwargs:
 863             # FORCE
 864             if "FORCE" in kwargs:
 865                 if kwargs["FORCE"].lower() != "false":  # if None or True set force to True
 866                     admin_query["force"] = True
 867                 del kwargs["FORCE"]
 868             # PUBLIC
 869             if "PUBLIC" in kwargs:
 870                 if kwargs["PUBLIC"].lower() != "false":  # if None or True set public to True
 871                     admin_query["public"] = True
 872                 else:
 873                     admin_query["public"] = False
 874                 del kwargs["PUBLIC"]
 875             # ADMIN
 876             if "ADMIN" in kwargs:
 877                 behave_as = kwargs.pop("ADMIN")
 878                 if behave_as.lower() != "false":
 879                     if not token_info["admin"]:
 880                         raise NbiException("Only admin projects can use 'ADMIN' query string", HTTPStatus.UNAUTHORIZED)
 881                     if not behave_as or behave_as.lower() == "true":  # convert True, None to empty list
 882                         admin_query["project_id"] = ()
 883                     elif isinstance(behave_as, (list, tuple)):
 884                         admin_query["project_id"] = behave_as
 885                     else:   # isinstance(behave_as, str)
 886                         admin_query["project_id"] = (behave_as, )
 887             if "SET_PROJECT" in kwargs:
 888                 set_project = kwargs.pop("SET_PROJECT")
 889                 if not set_project:
 890                     admin_query["set_project"] = list(admin_query["project_id"])
 891                 else:
 892                     if isinstance(set_project, str):
 893                         set_project = (set_project, )
 894                     if admin_query["project_id"]:
 895                         for p in set_project:
 896                             if p not in admin_query["project_id"]:
 897                                 raise NbiException("Unauthorized for 'SET_PROJECT={p}'. Try with 'ADMIN=True' or "
 898                                                    "'ADMIN='{p}'".format(p=p), HTTPStatus.UNAUTHORIZED)
 899                     admin_query["set_project"] = set_project
 900
 901             # PROJECT_READ
 902             # if "PROJECT_READ" in kwargs:
 903             #     admin_query["project"] = kwargs.pop("project")
 904             #     if admin_query["project"] == token_info["project_id"]:
 905         if method == "GET":
 906             if _id:
 907                 admin_query["method"] = "show"
 908             else:
 909                 admin_query["method"] = "list"
 910         elif method == "DELETE":
 911             admin_query["method"] = "delete"
 912         else:
 913             admin_query["method"] = "write"
 914         return admin_query
 915
 916     @cherrypy.expose
 917     def default(self, main_topic=None, version=None, topic=None, _id=None, item=None, *args, **kwargs):
 918         token_info = None
 919         outdata = None
 920         _format = None
 921         method = "DONE"
 922         engine_topic = None
 923         rollback = []
 924         engine_session = None
 925         try:
 926             if not main_topic or not version or not topic:
 927                 raise NbiException("URL must contain at least 'main_topic/version/topic'",
 928                                    HTTPStatus.METHOD_NOT_ALLOWED)
 929             if main_topic not in ("admin", "vnfpkgm", "nsd", "nslcm", "pdu", "nst", "nsilcm", "nspm"):
 930                 raise NbiException("URL main_topic '{}' not supported".format(main_topic),
 931                                    HTTPStatus.METHOD_NOT_ALLOWED)
 932             if version != 'v1':
 933                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 934
 935             if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
 936                 method = kwargs.pop("METHOD")
 937             else:
 938                 method = cherrypy.request.method
 939
 940             role_permission = self._check_valid_url_method(method, main_topic, version, topic, _id, item, *args)
 941             query_string_operations = self._extract_query_string_operations(kwargs, method)
 942             if main_topic == "admin" and topic == "tokens":
 943                 return self.token(method, _id, kwargs)
 944
 945             token_info = self.authenticator.authorize(role_permission, query_string_operations)
 946             engine_session = self._manage_admin_query(token_info, kwargs, method, _id)
 947             indata = self._format_in(kwargs)
 948             engine_topic = topic
 949             if topic == "subscriptions":
 950                 engine_topic = main_topic + "_" + topic
 951             if item and topic != "pm_jobs":
 952                 engine_topic = item
 953
 954             if main_topic == "nsd":
 955                 engine_topic = "nsds"
 956             elif main_topic == "vnfpkgm":
 957                 engine_topic = "vnfds"
 958             elif main_topic == "nslcm":
 959                 engine_topic = "nsrs"
 960                 if topic == "ns_lcm_op_occs":
 961                     engine_topic = "nslcmops"
 962                 if topic == "vnfrs" or topic == "vnf_instances":
 963                     engine_topic = "vnfrs"
 964             elif main_topic == "nst":
 965                 engine_topic = "nsts"
 966             elif main_topic == "nsilcm":
 967                 engine_topic = "nsis"
 968                 if topic == "nsi_lcm_op_occs":
 969                     engine_topic = "nsilcmops"
 970             elif main_topic == "pdu":
 971                 engine_topic = "pdus"
 972             if engine_topic == "vims":   # TODO this is for backward compatibility, it will be removed in the future
 973                 engine_topic = "vim_accounts"
 974
 975             if method == "GET":
 976                 if item in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd", "nst", "nst_content"):
 977                     if item in ("vnfd", "nsd", "nst"):
 978                         path = "$DESCRIPTOR"
 979                     elif args:
 980                         path = args
 981                     elif item == "artifacts":
 982                         path = ()
 983                     else:
 984                         path = None
 985                     file, _format = self.engine.get_file(engine_session, engine_topic, _id, path,
 986                                                          cherrypy.request.headers.get("Accept"))
 987                     outdata = file
 988                 elif not _id:
 989                     outdata = self.engine.get_item_list(engine_session, engine_topic, kwargs)
 990                 else:
 991                     if item == "reports":
 992                         # TODO check that project_id (_id in this context) has permissions
 993                         _id = args[0]
 994                     outdata = self.engine.get_item(engine_session, engine_topic, _id)
 995             elif method == "POST":
 996                 cherrypy.response.status = HTTPStatus.CREATED.value
 997                 if topic in ("ns_descriptors_content", "vnf_packages_content", "netslice_templates_content"):
 998                     _id = cherrypy.request.headers.get("Transaction-Id")
 999                     if not _id:
1000                         _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, {}, None,
1001                                                       cherrypy.request.headers)
1002                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1003                                                            cherrypy.request.headers)
1004                     if completed:
1005                         self._set_location_header(main_topic, version, topic, _id)
1006                     else:
1007                         cherrypy.response.headers["Transaction-Id"] = _id
1008                     outdata = {"id": _id}
1009                 elif topic == "ns_instances_content":
1010                     # creates NSR
1011                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1012                     # creates nslcmop
1013                     indata["lcmOperationType"] = "instantiate"
1014                     indata["nsInstanceId"] = _id
1015                     nslcmop_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, None)
1016                     self._set_location_header(main_topic, version, topic, _id)
1017                     outdata = {"id": _id, "nslcmop_id": nslcmop_id}
1018                 elif topic == "ns_instances" and item:
1019                     indata["lcmOperationType"] = item
1020                     indata["nsInstanceId"] = _id
1021                     _id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", indata, kwargs)
1022                     self._set_location_header(main_topic, version, "ns_lcm_op_occs", _id)
1023                     outdata = {"id": _id}
1024                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1025                 elif topic == "netslice_instances_content":
1026                     # creates NetSlice_Instance_record (NSIR)
1027                     _id, _ = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs)
1028                     self._set_location_header(main_topic, version, topic, _id)
1029                     indata["lcmOperationType"] = "instantiate"
1030                     indata["netsliceInstanceId"] = _id
1031                     nsilcmop_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1032                     outdata = {"id": _id, "nsilcmop_id": nsilcmop_id}
1033
1034                 elif topic == "netslice_instances" and item:
1035                     indata["lcmOperationType"] = item
1036                     indata["netsliceInstanceId"] = _id
1037                     _id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", indata, kwargs)
1038                     self._set_location_header(main_topic, version, "nsi_lcm_op_occs", _id)
1039                     outdata = {"id": _id}
1040                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1041                 else:
1042                     _id, op_id = self.engine.new_item(rollback, engine_session, engine_topic, indata, kwargs,
1043                                                       cherrypy.request.headers)
1044                     self._set_location_header(main_topic, version, topic, _id)
1045                     outdata = {"id": _id}
1046                     if op_id:
1047                         outdata["op_id"] = op_id
1048                         cherrypy.response.status = HTTPStatus.ACCEPTED.value
1049                     # TODO form NsdInfo when topic in ("ns_descriptors", "vnf_packages")
1050
1051             elif method == "DELETE":
1052                 if not _id:
1053                     outdata = self.engine.del_item_list(engine_session, engine_topic, kwargs)
1054                     cherrypy.response.status = HTTPStatus.OK.value
1055                 else:  # len(args) > 1
1056                     delete_in_process = False
1057                     if topic == "ns_instances_content" and not engine_session["force"]:
1058                         nslcmop_desc = {
1059                             "lcmOperationType": "terminate",
1060                             "nsInstanceId": _id,
1061                             "autoremove": True
1062                         }
1063                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nslcmops", nslcmop_desc, None)
1064                         if opp_id:
1065                             delete_in_process = True
1066                             outdata = {"_id": opp_id}
1067                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1068                     elif topic == "netslice_instances_content" and not engine_session["force"]:
1069                         nsilcmop_desc = {
1070                             "lcmOperationType": "terminate",
1071                             "netsliceInstanceId": _id,
1072                             "autoremove": True
1073                         }
1074                         opp_id, _ = self.engine.new_item(rollback, engine_session, "nsilcmops", nsilcmop_desc, None)
1075                         if opp_id:
1076                             delete_in_process = True
1077                             outdata = {"_id": opp_id}
1078                             cherrypy.response.status = HTTPStatus.ACCEPTED.value
1079                     if not delete_in_process:
1080                         self.engine.del_item(engine_session, engine_topic, _id)
1081                         cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1082                 if engine_topic in ("vim_accounts", "wim_accounts", "sdns"):
1083                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1084
1085             elif method in ("PUT", "PATCH"):
1086                 op_id = None
1087                 if not indata and not kwargs and not engine_session.get("set_project"):
1088                     raise NbiException("Nothing to update. Provide payload and/or query string",
1089                                        HTTPStatus.BAD_REQUEST)
1090                 if item in ("nsd_content", "package_content", "nst_content") and method == "PUT":
1091                     completed = self.engine.upload_content(engine_session, engine_topic, _id, indata, kwargs,
1092                                                            cherrypy.request.headers)
1093                     if not completed:
1094                         cherrypy.response.headers["Transaction-Id"] = id
1095                 else:
1096                     op_id = self.engine.edit_item(engine_session, engine_topic, _id, indata, kwargs)
1097
1098                 if op_id:
1099                     cherrypy.response.status = HTTPStatus.ACCEPTED.value
1100                     outdata = {"op_id": op_id}
1101                 else:
1102                     cherrypy.response.status = HTTPStatus.NO_CONTENT.value
1103                     outdata = None
1104             else:
1105                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
1106
1107             # if Role information changes, it is needed to reload the information of roles
1108             if topic == "roles" and method != "GET":
1109                 self.authenticator.load_operation_to_allowed_roles()
1110             return self._format_out(outdata, token_info, _format)
1111         except Exception as e:
1112             if isinstance(e, (NbiException, EngineException, DbException, FsException, MsgException, AuthException,
1113                               ValidationError, AuthconnException)):
1114                 http_code_value = cherrypy.response.status = e.http_code.value
1115                 http_code_name = e.http_code.name
1116                 cherrypy.log("Exception {}".format(e))
1117             else:
1118                 http_code_value = cherrypy.response.status = HTTPStatus.BAD_REQUEST.value  # INTERNAL_SERVER_ERROR
1119                 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
1120                 http_code_name = HTTPStatus.BAD_REQUEST.name
1121             if hasattr(outdata, "close"):  # is an open file
1122                 outdata.close()
1123             error_text = str(e)
1124             rollback.reverse()
1125             for rollback_item in rollback:
1126                 try:
1127                     if rollback_item.get("operation") == "set":
1128                         self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1129                                                rollback_item["content"], fail_on_empty=False)
1130                     else:
1131                         self.engine.db.del_one(rollback_item["topic"], {"_id": rollback_item["_id"]},
1132                                                fail_on_empty=False)
1133                 except Exception as e2:
1134                     rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2)
1135                     cherrypy.log(rollback_error_text)
1136                     error_text += ". " + rollback_error_text
1137             # if isinstance(e, MsgException):
1138             #     error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
1139             #         engine_topic[:-1], method, error_text)
1140             problem_details = {
1141                 "code": http_code_name,
1142                 "status": http_code_value,
1143                 "detail": error_text,
1144             }
1145             return self._format_out(problem_details, token_info)
1146             # raise cherrypy.HTTPError(e.http_code.value, str(e))
1147         finally:
1148             if token_info:
1149                 self._format_login(token_info)
1150                 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
1151                     for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
1152                         if outdata.get(logging_id):
1153                             cherrypy.request.login += ";{}={}".format(logging_id, outdata[logging_id][:36])
1154
1155
1156 def _start_service():
1157     """
1158     Callback function called when cherrypy.engine starts
1159     Override configuration with env variables
1160     Set database, storage, message configuration
1161     Init database with admin/admin user password
1162     """
1163     global nbi_server
1164     global subscription_thread
1165     cherrypy.log.error("Starting osm_nbi")
1166     # update general cherrypy configuration
1167     update_dict = {}
1168
1169     engine_config = cherrypy.tree.apps['/osm'].config
1170     for k, v in environ.items():
1171         if not k.startswith("OSMNBI_"):
1172             continue
1173         k1, _, k2 = k[7:].lower().partition("_")
1174         if not k2:
1175             continue
1176         try:
1177             # update static configuration
1178             if k == 'OSMNBI_STATIC_DIR':
1179                 engine_config["/static"]['tools.staticdir.dir'] = v
1180                 engine_config["/static"]['tools.staticdir.on'] = True
1181             elif k == 'OSMNBI_SOCKET_PORT' or k == 'OSMNBI_SERVER_PORT':
1182                 update_dict['server.socket_port'] = int(v)
1183             elif k == 'OSMNBI_SOCKET_HOST' or k == 'OSMNBI_SERVER_HOST':
1184                 update_dict['server.socket_host'] = v
1185             elif k1 in ("server", "test", "auth", "log"):
1186                 update_dict[k1 + '.' + k2] = v
1187             elif k1 in ("message", "database", "storage", "authentication"):
1188                 # k2 = k2.replace('_', '.')
1189                 if k2 in ("port", "db_port"):
1190                     engine_config[k1][k2] = int(v)
1191                 else:
1192                     engine_config[k1][k2] = v
1193
1194         except ValueError as e:
1195             cherrypy.log.error("Ignoring environ '{}': " + str(e))
1196         except Exception as e:
1197             cherrypy.log.warn("skipping environ '{}' on exception '{}'".format(k, e))
1198
1199     if update_dict:
1200         cherrypy.config.update(update_dict)
1201         engine_config["global"].update(update_dict)
1202
1203     # logging cherrypy
1204     log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
1205     log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
1206     logger_server = logging.getLogger("cherrypy.error")
1207     logger_access = logging.getLogger("cherrypy.access")
1208     logger_cherry = logging.getLogger("cherrypy")
1209     logger_nbi = logging.getLogger("nbi")
1210
1211     if "log.file" in engine_config["global"]:
1212         file_handler = logging.handlers.RotatingFileHandler(engine_config["global"]["log.file"],
1213                                                             maxBytes=100e6, backupCount=9, delay=0)
1214         file_handler.setFormatter(log_formatter_simple)
1215         logger_cherry.addHandler(file_handler)
1216         logger_nbi.addHandler(file_handler)
1217     # log always to standard output
1218     for format_, logger in {"nbi.server %(filename)s:%(lineno)s": logger_server,
1219                             "nbi.access %(filename)s:%(lineno)s": logger_access,
1220                             "%(name)s %(filename)s:%(lineno)s": logger_nbi
1221                             }.items():
1222         log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
1223         log_formatter_cherry = logging.Formatter(log_format_cherry, datefmt='%Y-%m-%dT%H:%M:%S')
1224         str_handler = logging.StreamHandler()
1225         str_handler.setFormatter(log_formatter_cherry)
1226         logger.addHandler(str_handler)
1227
1228     if engine_config["global"].get("log.level"):
1229         logger_cherry.setLevel(engine_config["global"]["log.level"])
1230         logger_nbi.setLevel(engine_config["global"]["log.level"])
1231
1232     # logging other modules
1233     for k1, logname in {"message": "nbi.msg", "database": "nbi.db", "storage": "nbi.fs"}.items():
1234         engine_config[k1]["logger_name"] = logname
1235         logger_module = logging.getLogger(logname)
1236         if "logfile" in engine_config[k1]:
1237             file_handler = logging.handlers.RotatingFileHandler(engine_config[k1]["logfile"],
1238                                                                 maxBytes=100e6, backupCount=9, delay=0)
1239             file_handler.setFormatter(log_formatter_simple)
1240             logger_module.addHandler(file_handler)
1241         if "loglevel" in engine_config[k1]:
1242             logger_module.setLevel(engine_config[k1]["loglevel"])
1243     # TODO add more entries, e.g.: storage
1244     cherrypy.tree.apps['/osm'].root.engine.start(engine_config)
1245     cherrypy.tree.apps['/osm'].root.authenticator.start(engine_config)
1246     cherrypy.tree.apps['/osm'].root.engine.init_db(target_version=database_version)
1247     cherrypy.tree.apps['/osm'].root.authenticator.init_db(target_version=auth_database_version)
1248
1249     # start subscriptions thread:
1250     subscription_thread = SubscriptionThread(config=engine_config, engine=nbi_server.engine)
1251     subscription_thread.start()
1252     # Do not capture except SubscriptionException
1253
1254     # load and print version. Ignore possible errors, e.g. file not found
1255     try:
1256         with open("{}/version".format(engine_config["/static"]['tools.staticdir.dir'])) as version_file:
1257             version_data = version_file.read()
1258             version = version_data.replace("\n", " ")
1259             backend = engine_config["authentication"]["backend"]
1260             cherrypy.log.error("Starting OSM NBI Version {} with {} authentication backend"
1261                                .format(version, backend))
1262     except Exception:
1263         pass
1264
1265
1266 def _stop_service():
1267     """
1268     Callback function called when cherrypy.engine stops
1269     TODO: Ending database connections.
1270     """
1271     global subscription_thread
1272     if subscription_thread:
1273         subscription_thread.terminate()
1274     subscription_thread = None
1275     cherrypy.tree.apps['/osm'].root.engine.stop()
1276     cherrypy.log.error("Stopping osm_nbi")
1277
1278
1279 def nbi(config_file):
1280     global nbi_server
1281     # conf = {
1282     #     '/': {
1283     #         #'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
1284     #         'tools.sessions.on': True,
1285     #         'tools.response_headers.on': True,
1286     #         # 'tools.response_headers.headers': [('Content-Type', 'text/plain')],
1287     #     }
1288     # }
1289     # cherrypy.Server.ssl_module = 'builtin'
1290     # cherrypy.Server.ssl_certificate = "http/cert.pem"
1291     # cherrypy.Server.ssl_private_key = "http/privkey.pem"
1292     # cherrypy.Server.thread_pool = 10
1293     # cherrypy.config.update({'Server.socket_port': config["port"], 'Server.socket_host': config["host"]})
1294
1295     # cherrypy.config.update({'tools.auth_basic.on': True,
1296     #    'tools.auth_basic.realm': 'localhost',
1297     #    'tools.auth_basic.checkpassword': validate_password})
1298     nbi_server = Server()
1299     cherrypy.engine.subscribe('start', _start_service)
1300     cherrypy.engine.subscribe('stop', _stop_service)
1301     cherrypy.quickstart(nbi_server, '/osm', config_file)
1302
1303
1304 def usage():
1305     print("""Usage: {} [options]
1306         -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
1307         -h|--help: shows this help
1308         """.format(sys.argv[0]))
1309     # --log-socket-host HOST: send logs to this host")
1310     # --log-socket-port PORT: send logs using this port (default: 9022)")
1311
1312
1313 if __name__ == '__main__':
1314     try:
1315         # load parameters and configuration
1316         opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
1317         # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
1318         config_file = None
1319         for o, a in opts:
1320             if o in ("-h", "--help"):
1321                 usage()
1322                 sys.exit()
1323             elif o in ("-c", "--config"):
1324                 config_file = a
1325             # elif o == "--log-socket-port":
1326             #     log_socket_port = a
1327             # elif o == "--log-socket-host":
1328             #     log_socket_host = a
1329             # elif o == "--log-file":
1330             #     log_file = a
1331             else:
1332                 assert False, "Unhandled option"
1333         if config_file:
1334             if not path.isfile(config_file):
1335                 print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
1336                 exit(1)
1337         else:
1338             for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./nbi.cfg", "/etc/osm/nbi.cfg"):
1339                 if path.isfile(config_file):
1340                     break
1341             else:
1342                 print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
1343                 exit(1)
1344         nbi(config_file)
1345     except getopt.GetoptError as e:
1346         print(str(e), file=sys.stderr)
1347         # usage()
1348         exit(1)