osm_nbi/engine.py

   1 # -*- coding: utf-8 -*-
   2
   3 from osm_common import dbmongo
   4 from osm_common import dbmemory
   5 from osm_common import fslocal
   6 from osm_common import msglocal
   7 from osm_common import msgkafka
   8 import tarfile
   9 import yaml
  10 import json
  11 import logging
  12 from random import choice as random_choice
  13 from uuid import uuid4
  14 from hashlib import sha256, md5
  15 from osm_common.dbbase import DbException, deep_update
  16 from osm_common.fsbase import FsException
  17 from osm_common.msgbase import MsgException
  18 from http import HTTPStatus
  19 from time import time
  20 from copy import deepcopy, copy
  21 from validation import validate_input, ValidationError
  22
  23 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  24
  25
  26 class EngineException(Exception):
  27
  28     def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
  29         self.http_code = http_code
  30         Exception.__init__(self, message)
  31
  32
  33 def get_iterable(input):
  34     """
  35     Returns an iterable, in case input is None it just returns an empty tuple
  36     :param input:
  37     :return: iterable
  38     """
  39     if input is None:
  40         return ()
  41     return input
  42
  43
  44 class Engine(object):
  45
  46     def __init__(self):
  47         self.tokens = {}
  48         self.db = None
  49         self.fs = None
  50         self.msg = None
  51         self.config = None
  52         self.logger = logging.getLogger("nbi.engine")
  53
  54     def start(self, config):
  55         """
  56         Connect to database, filesystem storage, and messaging
  57         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  58         :return: None
  59         """
  60         self.config = config
  61         try:
  62             if not self.db:
  63                 if config["database"]["driver"] == "mongo":
  64                     self.db = dbmongo.DbMongo()
  65                     self.db.db_connect(config["database"])
  66                 elif config["database"]["driver"] == "memory":
  67                     self.db = dbmemory.DbMemory()
  68                     self.db.db_connect(config["database"])
  69                 else:
  70                     raise EngineException("Invalid configuration param '{}' at '[database]':'driver'".format(
  71                         config["database"]["driver"]))
  72             if not self.fs:
  73                 if config["storage"]["driver"] == "local":
  74                     self.fs = fslocal.FsLocal()
  75                     self.fs.fs_connect(config["storage"])
  76                 else:
  77                     raise EngineException("Invalid configuration param '{}' at '[storage]':'driver'".format(
  78                         config["storage"]["driver"]))
  79             if not self.msg:
  80                 if config["message"]["driver"] == "local":
  81                     self.msg = msglocal.MsgLocal()
  82                     self.msg.connect(config["message"])
  83                 elif config["message"]["driver"] == "kafka":
  84                     self.msg = msgkafka.MsgKafka()
  85                     self.msg.connect(config["message"])
  86                 else:
  87                     raise EngineException("Invalid configuration param '{}' at '[message]':'driver'".format(
  88                         config["storage"]["driver"]))
  89         except (DbException, FsException, MsgException) as e:
  90             raise EngineException(str(e), http_code=e.http_code)
  91
  92     def stop(self):
  93         try:
  94             if self.db:
  95                 self.db.db_disconnect()
  96             if self.fs:
  97                 self.fs.fs_disconnect()
  98             if self.fs:
  99                 self.fs.fs_disconnect()
 100         except (DbException, FsException, MsgException) as e:
 101             raise EngineException(str(e), http_code=e.http_code)
 102
 103     def authorize(self, token):
 104         try:
 105             if not token:
 106                 raise EngineException("Needed a token or Authorization http header",
 107                                       http_code=HTTPStatus.UNAUTHORIZED)
 108             if token not in self.tokens:
 109                 raise EngineException("Invalid token or Authorization http header",
 110                                       http_code=HTTPStatus.UNAUTHORIZED)
 111             session = self.tokens[token]
 112             now = time()
 113             if session["expires"] < now:
 114                 del self.tokens[token]
 115                 raise EngineException("Expired Token or Authorization http header",
 116                                       http_code=HTTPStatus.UNAUTHORIZED)
 117             return session
 118         except EngineException:
 119             if self.config["global"].get("test.user_not_authorized"):
 120                 return {"id": "fake-token-id-for-test",
 121                         "project_id": self.config["global"].get("test.project_not_authorized", "admin"),
 122                         "username": self.config["global"]["test.user_not_authorized"]}
 123             else:
 124                 raise
 125
 126     def new_token(self, session, indata, remote):
 127         now = time()
 128         user_content = None
 129
 130         # Try using username/password
 131         if indata.get("username"):
 132             user_rows = self.db.get_list("users", {"username": indata.get("username")})
 133             user_content = None
 134             if user_rows:
 135                 user_content = user_rows[0]
 136                 salt = user_content["_admin"]["salt"]
 137                 shadow_password = sha256(indata.get("password", "").encode('utf-8') + salt.encode('utf-8')).hexdigest()
 138                 if shadow_password != user_content["password"]:
 139                     user_content = None
 140             if not user_content:
 141                 raise EngineException("Invalid username/password", http_code=HTTPStatus.UNAUTHORIZED)
 142         elif session:
 143             user_rows = self.db.get_list("users", {"username": session["username"]})
 144             if user_rows:
 145                 user_content = user_rows[0]
 146             else:
 147                 raise EngineException("Invalid token", http_code=HTTPStatus.UNAUTHORIZED)
 148         else:
 149             raise EngineException("Provide credentials: username/password or Authorization Bearer token",
 150                                   http_code=HTTPStatus.UNAUTHORIZED)
 151
 152         token_id = ''.join(random_choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
 153                            for _ in range(0, 32))
 154         if indata.get("project_id"):
 155             project_id = indata.get("project_id")
 156             if project_id not in user_content["projects"]:
 157                 raise EngineException("project {} not allowed for this user".format(project_id),
 158                                       http_code=HTTPStatus.UNAUTHORIZED)
 159         else:
 160             project_id = user_content["projects"][0]
 161         if project_id == "admin":
 162             session_admin = True
 163         else:
 164             project = self.db.get_one("projects", {"_id": project_id})
 165             session_admin = project.get("admin", False)
 166         new_session = {"issued_at": now, "expires": now+3600,
 167                        "_id": token_id, "id": token_id, "project_id": project_id, "username": user_content["username"],
 168                        "remote_port": remote.port, "admin": session_admin}
 169         if remote.name:
 170             new_session["remote_host"] = remote.name
 171         elif remote.ip:
 172             new_session["remote_host"] = remote.ip
 173
 174         self.tokens[token_id] = new_session
 175         return deepcopy(new_session)
 176
 177     def get_token_list(self, session):
 178         token_list = []
 179         for token_id, token_value in self.tokens.items():
 180             if token_value["username"] == session["username"]:
 181                 token_list.append(deepcopy(token_value))
 182         return token_list
 183
 184     def get_token(self, session, token_id):
 185         token_value = self.tokens.get(token_id)
 186         if not token_value:
 187             raise EngineException("token not found", http_code=HTTPStatus.NOT_FOUND)
 188         if token_value["username"] != session["username"] and not session["admin"]:
 189             raise EngineException("needed admin privileges", http_code=HTTPStatus.UNAUTHORIZED)
 190         return token_value
 191
 192     def del_token(self, token_id):
 193         try:
 194             del self.tokens[token_id]
 195             return "token '{}' deleted".format(token_id)
 196         except KeyError:
 197             raise EngineException("Token '{}' not found".format(token_id), http_code=HTTPStatus.NOT_FOUND)
 198
 199     @staticmethod
 200     def _remove_envelop(item, indata=None):
 201         """
 202         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
 203         vnfd or nsd content
 204         :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds
 205         :param indata: Content to be inspected
 206         :return: the useful part of indata
 207         """
 208         clean_indata = indata
 209         if not indata:
 210             return {}
 211         if item == "vnfds":
 212             if clean_indata.get('vnfd:vnfd-catalog'):
 213                 clean_indata = clean_indata['vnfd:vnfd-catalog']
 214             elif clean_indata.get('vnfd-catalog'):
 215                 clean_indata = clean_indata['vnfd-catalog']
 216             if clean_indata.get('vnfd'):
 217                 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 218                     raise EngineException("'vnfd' must be a list only one element")
 219                 clean_indata = clean_indata['vnfd'][0]
 220         elif item == "nsds":
 221             if clean_indata.get('nsd:nsd-catalog'):
 222                 clean_indata = clean_indata['nsd:nsd-catalog']
 223             elif clean_indata.get('nsd-catalog'):
 224                 clean_indata = clean_indata['nsd-catalog']
 225             if clean_indata.get('nsd'):
 226                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 227                     raise EngineException("'nsd' must be a list only one element")
 228                 clean_indata = clean_indata['nsd'][0]
 229         elif item == "userDefinedData":
 230             if "userDefinedData" in indata:
 231                 clean_indata = clean_indata['userDefinedData']
 232         return clean_indata
 233
 234     def _check_project_dependencies(self, project_id):
 235         """
 236         Check if a project can be deleted
 237         :param session:
 238         :param _id:
 239         :return:
 240         """
 241         # TODO Is it needed to check descriptors _admin.project_read/project_write??
 242         _filter = {"projects": project_id}
 243         if self.db.get_list("users", _filter):
 244             raise EngineException("There are users that uses this project", http_code=HTTPStatus.CONFLICT)
 245
 246     def _check_dependencies_on_descriptor(self, session, item, descriptor_id, _id):
 247         """
 248         Check that the descriptor to be deleded is not a dependency of others
 249         :param session: client session information
 250         :param item: can be vnfds, nsds
 251         :param descriptor_id: id (provided by client) of descriptor to be deleted
 252         :param _id: internal id of descriptor to be deleted
 253         :return: None or raises exception
 254         """
 255         if item == "vnfds":
 256             _filter = {"constituent-vnfd.ANYINDEX.vnfd-id-ref": descriptor_id}
 257             if self.get_item_list(session, "nsds", _filter):
 258                 raise EngineException("There are nsd that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 259             if self.get_item_list(session, "vnfrs", {"vnfd-id": _id}):
 260                 raise EngineException("There are vnfr that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 261         elif item == "nsds":
 262             _filter = {"nsdId": _id}
 263             if self.get_item_list(session, "nsrs", _filter):
 264                 raise EngineException("There are nsr that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 265
 266     def _check_descriptor_dependencies(self, session, item, descriptor):
 267         """
 268         Check that the dependent descriptors exist on a new descriptor or edition
 269         :param session: client session information
 270         :param item: can be nsds, nsrs
 271         :param descriptor: descriptor to be inserted or edit
 272         :return: None or raises exception
 273         """
 274         if item == "nsds":
 275             if not descriptor.get("constituent-vnfd"):
 276                 return
 277             for vnf in descriptor["constituent-vnfd"]:
 278                 vnfd_id = vnf["vnfd-id-ref"]
 279                 if not self.get_item_list(session, "vnfds", {"id": vnfd_id}):
 280                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 281                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 282         elif item == "nsrs":
 283             if not descriptor.get("nsdId"):
 284                 return
 285             nsd_id = descriptor["nsdId"]
 286             if not self.get_item_list(session, "nsds", {"id": nsd_id}):
 287                 raise EngineException("Descriptor error at nsdId='{}' references a non exist nsd".format(nsd_id),
 288                                       http_code=HTTPStatus.CONFLICT)
 289
 290     def _check_edition(self, session, item, indata, id, force=False):
 291         if item == "users":
 292             if indata.get("projects"):
 293                 if not session["admin"]:
 294                     raise EngineException("Needed admin privileges to edit user projects", HTTPStatus.UNAUTHORIZED)
 295             if indata.get("password"):
 296                 # regenerate salt and encrypt password
 297                 salt = uuid4().hex
 298                 indata["_admin"] = {"salt": salt}
 299                 indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
 300
 301     def _validate_new_data(self, session, item, indata, id=None, force=False):
 302         if item == "users":
 303             # check username not exists
 304             if not id and self.db.get_one(item, {"username": indata.get("username")}, fail_on_empty=False,
 305                                           fail_on_more=False):
 306                 raise EngineException("username '{}' exists".format(indata["username"]), HTTPStatus.CONFLICT)
 307             # check projects
 308             if not force:
 309                 for p in indata["projects"]:
 310                     if p == "admin":
 311                         continue
 312                     if not self.db.get_one("projects", {"_id": p}, fail_on_empty=False, fail_on_more=False):
 313                         raise EngineException("project '{}' does not exists".format(p), HTTPStatus.CONFLICT)
 314         elif item == "projects":
 315             if not indata.get("name"):
 316                 raise EngineException("missing 'name'")
 317             # check name not exists
 318             if not id and self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
 319                 raise EngineException("name '{}' exists".format(indata["name"]), HTTPStatus.CONFLICT)
 320         elif item in ("vnfds", "nsds"):
 321             filter = {"id": indata["id"]}
 322             if id:
 323                 filter["_id.neq"] = id
 324             # TODO add admin to filter, validate rights
 325             self._add_read_filter(session, item, filter)
 326             if self.db.get_one(item, filter, fail_on_empty=False):
 327                 raise EngineException("{} with id '{}' already exists for this tenant".format(item[:-1], indata["id"]),
 328                                       HTTPStatus.CONFLICT)
 329             # TODO validate with pyangbind. Load and dumps to convert data types
 330             if item == "nsds":
 331                 # transform constituent-vnfd:member-vnf-index to string
 332                 if indata.get("constituent-vnfd"):
 333                     for constituent_vnfd in indata["constituent-vnfd"]:
 334                         if "member-vnf-index" in constituent_vnfd:
 335                             constituent_vnfd["member-vnf-index"] = str(constituent_vnfd["member-vnf-index"])
 336
 337             if item == "nsds" and not force:
 338                 self._check_descriptor_dependencies(session, "nsds", indata)
 339         elif item == "userDefinedData":
 340             # TODO validate userDefinedData is a keypair values
 341             pass
 342
 343         elif item == "nsrs":
 344             pass
 345         elif item == "vim_accounts" or item == "sdns":
 346             filter = {"name": indata.get("name")}
 347             if id:
 348                 filter["_id.neq"] = id
 349             if self.db.get_one(item, filter, fail_on_empty=False, fail_on_more=False):
 350                 raise EngineException("name '{}' already exists for {}".format(indata["name"], item),
 351                                       HTTPStatus.CONFLICT)
 352
 353     def _check_ns_operation(self, session, nsr, operation, indata):
 354         """
 355         Check that user has enter right parameters for the operation
 356         :param session:
 357         :param operation: it can be: instantiate, terminate, action, TODO: update, heal
 358         :param indata: descriptor with the parameters of the operation
 359         :return: None
 360         """
 361         vnfds = {}
 362         vim_accounts = []
 363         nsd = nsr["nsd"]
 364
 365         def check_valid_vnf_member_index(member_vnf_index):
 366             for vnf in nsd["constituent-vnfd"]:
 367                 if member_vnf_index == vnf["member-vnf-index"]:
 368                     vnfd_id = vnf["vnfd-id-ref"]
 369                     if vnfd_id not in vnfds:
 370                         vnfds[vnfd_id] = self.db.get_one("vnfds", {"id": vnfd_id})
 371                     return vnfds[vnfd_id]
 372             else:
 373                 raise EngineException("Invalid parameter member_vnf_index='{}' is not one of the "
 374                                       "nsd:constituent-vnfd".format(member_vnf_index))
 375
 376         def check_valid_vim_account(vim_account):
 377             if vim_account in vim_accounts:
 378                 return
 379             try:
 380                 self.db.get_one("vim_accounts", {"_id": vim_account})
 381             except Exception:
 382                 raise EngineException("Invalid vimAccountId='{}' not present".format(vim_account))
 383             vim_accounts.append(vim_account)
 384
 385         if operation == "action":
 386             # check vnf_member_index
 387             if indata.get("vnf_member_index"):
 388                 indata["member_vnf_index"] = indata.pop("vnf_member_index")    # for backward compatibility
 389             if not indata.get("member_vnf_index"):
 390                 raise EngineException("Missing 'member_vnf_index' parameter")
 391             vnfd = check_valid_vnf_member_index(indata["member_vnf_index"])
 392             # check primitive
 393             for config_primitive in get_iterable(vnfd.get("vnf-configuration", {}).get("config-primitive")):
 394                 if indata["primitive"] == config_primitive["name"]:
 395                     # check needed primitive_params are provided
 396                     if indata.get("primitive_params"):
 397                         in_primitive_params_copy = copy(indata["primitive_params"])
 398                     else:
 399                         in_primitive_params_copy = {}
 400                     for paramd in get_iterable(config_primitive.get("parameter")):
 401                         if paramd["name"] in in_primitive_params_copy:
 402                             del in_primitive_params_copy[paramd["name"]]
 403                         elif not paramd.get("default-value"):
 404                             raise EngineException("Needed parameter {} not provided for primitive '{}'".format(
 405                                 paramd["name"], indata["primitive"]))
 406                     # check no extra primitive params are provided
 407                     if in_primitive_params_copy:
 408                         raise EngineException("parameter/s '{}' not present at vnfd for primitive '{}'".format(
 409                             list(in_primitive_params_copy.keys()), indata["primitive"]))
 410                     break
 411             else:
 412                 raise EngineException("Invalid primitive '{}' is not present at vnfd".format(indata["primitive"]))
 413         if operation == "scale":
 414             vnfd = check_valid_vnf_member_index(indata["scaleVnfData"]["scaleByStepData"]["member-vnf-index"])
 415             for scaling_group in get_iterable(vnfd.get("scaling-group-descriptor")):
 416                 if indata["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"] == scaling_group["name"]:
 417                     break
 418             else:
 419                 raise EngineException("Invalid scaleVnfData:scaleByStepData:scaling-group-descriptor '{}' is not "
 420                                       "present at vnfd:scaling-group-descriptor".format(
 421                                           indata["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]))
 422         if operation == "instantiate":
 423             # check vim_account
 424             check_valid_vim_account(indata["vimAccountId"])
 425             for in_vnf in get_iterable(indata.get("vnf")):
 426                 vnfd = check_valid_vnf_member_index(in_vnf["member-vnf-index"])
 427                 if in_vnf.get("vimAccountId"):
 428                     check_valid_vim_account(in_vnf["vimAccountId"])
 429                 for in_vdu in get_iterable(in_vnf.get("vdu")):
 430                     for vdud in get_iterable(vnfd.get("vdu")):
 431                         if vdud["id"] == in_vdu["id"]:
 432                             for volume in get_iterable(in_vdu.get("volume")):
 433                                 for volumed in get_iterable(vdud.get("volumes")):
 434                                     if volumed["name"] == volume["name"]:
 435                                         break
 436                                 else:
 437                                     raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:vdu[id='{}']:"
 438                                                           "volume:name='{}' is not present at vnfd:vdu:volumes list".
 439                                                           format(in_vnf["member-vnf-index"], in_vdu["id"],
 440                                                                  volume["name"]))
 441                             break
 442                     else:
 443                         raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:vdu:id='{}' is not "
 444                                               "present at vnfd".format(in_vnf["member-vnf-index"], in_vdu["id"]))
 445
 446                 for in_internal_vld in get_iterable(in_vnf.get("internal-vld")):
 447                     for internal_vldd in get_iterable(vnfd.get("internal-vld")):
 448                         if in_internal_vld["name"] == internal_vldd["name"] or \
 449                                 in_internal_vld["name"] == internal_vldd["id"]:
 450                             break
 451                     else:
 452                         raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:internal-vld:name='{}'"
 453                                               " is not present at vnfd '{}'".format(in_vnf["member-vnf-index"],
 454                                                                                     in_internal_vld["name"],
 455                                                                                     vnfd["id"]))
 456             for in_vld in get_iterable(indata.get("vld")):
 457                 for vldd in get_iterable(nsd.get("vld")):
 458                     if in_vld["name"] == vldd["name"] or in_vld["name"] == vldd["id"]:
 459                         break
 460                 else:
 461                     raise EngineException("Invalid parameter vld:name='{}' is not present at nsd:vld".format(
 462                         in_vld["name"]))
 463
 464     def _format_new_data(self, session, item, indata):
 465         now = time()
 466         if "_admin" not in indata:
 467             indata["_admin"] = {}
 468         indata["_admin"]["created"] = now
 469         indata["_admin"]["modified"] = now
 470         if item == "users":
 471             indata["_id"] = indata["username"]
 472             salt = uuid4().hex
 473             indata["_admin"]["salt"] = salt
 474             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
 475         elif item == "projects":
 476             indata["_id"] = indata["name"]
 477         else:
 478             if not indata.get("_id"):
 479                 indata["_id"] = str(uuid4())
 480             if item in ("vnfds", "nsds", "nsrs", "vnfrs"):
 481                 if not indata["_admin"].get("projects_read"):
 482                     indata["_admin"]["projects_read"] = [session["project_id"]]
 483                 if not indata["_admin"].get("projects_write"):
 484                     indata["_admin"]["projects_write"] = [session["project_id"]]
 485             if item in ("vnfds", "nsds"):
 486                 indata["_admin"]["onboardingState"] = "CREATED"
 487                 indata["_admin"]["operationalState"] = "DISABLED"
 488                 indata["_admin"]["usageSate"] = "NOT_IN_USE"
 489             if item == "nsrs":
 490                 indata["_admin"]["nsState"] = "NOT_INSTANTIATED"
 491             if item in ("vim_accounts", "sdns"):
 492                 indata["_admin"]["operationalState"] = "PROCESSING"
 493
 494     def upload_content(self, session, item, _id, indata, kwargs, headers):
 495         """
 496         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 497         :param session: session
 498         :param item: can be nsds or vnfds
 499         :param _id : the nsd,vnfd is already created, this is the id
 500         :param indata: http body request
 501         :param kwargs: user query string to override parameters. NOT USED
 502         :param headers:  http request headers
 503         :return: True package has is completely uploaded or False if partial content has been uplodaed.
 504             Raise exception on error
 505         """
 506         # Check that _id exists and it is valid
 507         current_desc = self.get_item(session, item, _id)
 508
 509         content_range_text = headers.get("Content-Range")
 510         expected_md5 = headers.get("Content-File-MD5")
 511         compressed = None
 512         content_type = headers.get("Content-Type")
 513         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 514                 "application/zip" in content_type:
 515             compressed = "gzip"
 516         filename = headers.get("Content-Filename")
 517         if not filename:
 518             filename = "package.tar.gz" if compressed else "package"
 519         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 520         file_pkg = None
 521         error_text = ""
 522         try:
 523             if content_range_text:
 524                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 525                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 526                     raise IndexError()
 527                 start = int(content_range[1])
 528                 end = int(content_range[2]) + 1
 529                 total = int(content_range[3])
 530             else:
 531                 start = 0
 532
 533             if start:
 534                 if not self.fs.file_exists(_id, 'dir'):
 535                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 536             else:
 537                 self.fs.file_delete(_id, ignore_non_exist=True)
 538                 self.fs.mkdir(_id)
 539
 540             storage = self.fs.get_params()
 541             storage["folder"] = _id
 542
 543             file_path = (_id, filename)
 544             if self.fs.file_exists(file_path, 'file'):
 545                 file_size = self.fs.file_size(file_path)
 546             else:
 547                 file_size = 0
 548             if file_size != start:
 549                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 550                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 551             file_pkg = self.fs.file_open(file_path, 'a+b')
 552             if isinstance(indata, dict):
 553                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 554                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 555             else:
 556                 indata_len = 0
 557                 while True:
 558                     indata_text = indata.read(4096)
 559                     indata_len += len(indata_text)
 560                     if not indata_text:
 561                         break
 562                     file_pkg.write(indata_text)
 563             if content_range_text:
 564                 if indata_len != end-start:
 565                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 566                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 567                 if end != total:
 568                     # TODO update to UPLOADING
 569                     return False
 570
 571             # PACKAGE UPLOADED
 572             if expected_md5:
 573                 file_pkg.seek(0, 0)
 574                 file_md5 = md5()
 575                 chunk_data = file_pkg.read(1024)
 576                 while chunk_data:
 577                     file_md5.update(chunk_data)
 578                     chunk_data = file_pkg.read(1024)
 579                 if expected_md5 != file_md5.hexdigest():
 580                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 581             file_pkg.seek(0, 0)
 582             if compressed == "gzip":
 583                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 584                 descriptor_file_name = None
 585                 for tarinfo in tar:
 586                     tarname = tarinfo.name
 587                     tarname_path = tarname.split("/")
 588                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 589                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 590                     if len(tarname_path) == 1 and not tarinfo.isdir():
 591                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 592                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 593                         storage["pkg-dir"] = tarname_path[0]
 594                         if len(tarname_path) == 2:
 595                             if descriptor_file_name:
 596                                 raise EngineException(
 597                                     "Found more than one descriptor file at package descriptor tar.gz")
 598                             descriptor_file_name = tarname
 599                 if not descriptor_file_name:
 600                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 601                 storage["descriptor"] = descriptor_file_name
 602                 storage["zipfile"] = filename
 603                 self.fs.file_extract(tar, _id)
 604                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 605                     content = descriptor_file.read()
 606             else:
 607                 content = file_pkg.read()
 608                 storage["descriptor"] = descriptor_file_name = filename
 609
 610             if descriptor_file_name.endswith(".json"):
 611                 error_text = "Invalid json format "
 612                 indata = json.load(content)
 613             else:
 614                 error_text = "Invalid yaml format "
 615                 indata = yaml.load(content)
 616
 617             current_desc["_admin"]["storage"] = storage
 618             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 619             current_desc["_admin"]["operationalState"] = "ENABLED"
 620
 621             self._edit_item(session, item, _id, current_desc, indata, kwargs)
 622             # TODO if descriptor has changed because kwargs update content and remove cached zip
 623             # TODO if zip is not present creates one
 624             return True
 625
 626         except EngineException:
 627             raise
 628         except IndexError:
 629             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 630                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 631         except IOError as e:
 632             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 633         except tarfile.ReadError as e:
 634             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 635         except (ValueError, yaml.YAMLError) as e:
 636             raise EngineException(error_text + str(e))
 637         finally:
 638             if file_pkg:
 639                 file_pkg.close()
 640
 641     def new_nsr(self, rollback, session, ns_request):
 642         """
 643         Creates a new nsr into database. It also creates needed vnfrs
 644         :param rollback: list where this method appends created items at database in case a rollback may to be done
 645         :param session: contains the used login username and working project
 646         :param ns_request: params to be used for the nsr
 647         :return: the _id of nsr descriptor stored at database
 648         """
 649         rollback_index = len(rollback)
 650         step = ""
 651         try:
 652             # look for nsr
 653             step = "getting nsd id='{}' from database".format(ns_request.get("nsdId"))
 654             nsd = self.get_item(session, "nsds", ns_request["nsdId"])
 655             nsr_id = str(uuid4())
 656             now = time()
 657             step = "filling nsr from input data"
 658             nsr_descriptor = {
 659                 "name": ns_request["nsName"],
 660                 "name-ref": ns_request["nsName"],
 661                 "short-name": ns_request["nsName"],
 662                 "admin-status": "ENABLED",
 663                 "nsd": nsd,
 664                 "datacenter": ns_request["vimAccountId"],
 665                 "resource-orchestrator": "osmopenmano",
 666                 "description": ns_request.get("nsDescription", ""),
 667                 "constituent-vnfr-ref": [],
 668
 669                 "operational-status": "init",    # typedef ns-operational-
 670                 "config-status": "init",         # typedef config-states
 671                 "detailed-status": "scheduled",
 672
 673                 "orchestration-progress": {},
 674                 # {"networks": {"active": 0, "total": 0}, "vms": {"active": 0, "total": 0}},
 675
 676                 "create-time": now,
 677                 "nsd-name-ref": nsd["name"],
 678                 "operational-events": [],   # "id", "timestamp", "description", "event",
 679                 "nsd-ref": nsd["id"],
 680                 "nsdId": nsd["_id"],
 681                 "instantiate_params": ns_request,
 682                 "ns-instance-config-ref": nsr_id,
 683                 "id": nsr_id,
 684                 "_id": nsr_id,
 685                 # "input-parameter": xpath, value,
 686                 "ssh-authorized-key": ns_request.get("key-pair-ref"),
 687             }
 688             ns_request["nsr_id"] = nsr_id
 689
 690             # Create VNFR
 691             needed_vnfds = {}
 692             for member_vnf in nsd["constituent-vnfd"]:
 693                 vnfd_id = member_vnf["vnfd-id-ref"]
 694                 step = "getting vnfd id='{}' constituent-vnfd='{}' from database".format(
 695                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 696                 if vnfd_id not in needed_vnfds:
 697                     # Obtain vnfd
 698                     vnf_filter = {"id": vnfd_id}
 699                     self._add_read_filter(session, "vnfds", vnf_filter)
 700                     vnfd = self.db.get_one("vnfds", vnf_filter)
 701                     vnfd.pop("_admin")
 702                     needed_vnfds[vnfd_id] = vnfd
 703                 else:
 704                     vnfd = needed_vnfds[vnfd_id]
 705                 step = "filling vnfr  vnfd-id='{}' constituent-vnfd='{}'".format(
 706                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 707                 vnfr_id = str(uuid4())
 708                 vnfr_descriptor = {
 709                     "id": vnfr_id,
 710                     "_id": vnfr_id,
 711                     "nsr-id-ref": nsr_id,
 712                     "member-vnf-index-ref": member_vnf["member-vnf-index"],
 713                     "created-time": now,
 714                     # "vnfd": vnfd,        # at OSM model.but removed to avoid data duplication TODO: revise
 715                     "vnfd-ref": vnfd_id,
 716                     "vnfd-id": vnfd["_id"],    # not at OSM model, but useful
 717                     "vim-account-id": None,
 718                     "vdur": [],
 719                     "connection-point": [],
 720                     "ip-address": None,  # mgmt-interface filled by LCM
 721                 }
 722                 for cp in vnfd.get("connection-point", ()):
 723                     vnf_cp = {
 724                         "name": cp["name"],
 725                         "connection-point-id": cp.get("id"),
 726                         "id": cp.get("id"),
 727                         # "ip-address", "mac-address" # filled by LCM
 728                         # vim-id  # TODO it would be nice having a vim port id
 729                     }
 730                     vnfr_descriptor["connection-point"].append(vnf_cp)
 731                 for vdu in vnfd["vdu"]:
 732                     vdur_id = str(uuid4())
 733                     vdur = {
 734                         "id": vdur_id,
 735                         "vdu-id-ref": vdu["id"],
 736                         # TODO      "name": ""     Name of the VDU in the VIM
 737                         "ip-address": None,  # mgmt-interface filled by LCM
 738                         # "vim-id", "flavor-id", "image-id", "management-ip" # filled by LCM
 739                         "internal-connection-point": [],
 740                         "interfaces": [],
 741                     }
 742                     # TODO volumes: name, volume-id
 743                     for icp in vdu.get("internal-connection-point", ()):
 744                         vdu_icp = {
 745                             "id": icp["id"],
 746                             "connection-point-id": icp["id"],
 747                             "name": icp.get("name"),
 748                             # "ip-address", "mac-address" # filled by LCM
 749                             # vim-id  # TODO it would be nice having a vim port id
 750                         }
 751                         vdur["internal-connection-point"].append(vdu_icp)
 752                     for iface in vdu.get("interface", ()):
 753                         vdu_iface = {
 754                             "name": iface.get("name"),
 755                             # "ip-address", "mac-address" # filled by LCM
 756                             # vim-id  # TODO it would be nice having a vim port id
 757                         }
 758                         vdur["interfaces"].append(vdu_iface)
 759                     vnfr_descriptor["vdur"].append(vdur)
 760
 761                 step = "creating vnfr vnfd-id='{}' constituent-vnfd='{}' at database".format(
 762                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 763                 self._format_new_data(session, "vnfrs", vnfr_descriptor)
 764                 self.db.create("vnfrs", vnfr_descriptor)
 765                 rollback.insert(0, {"item": "vnfrs", "_id": vnfr_id})
 766                 nsr_descriptor["constituent-vnfr-ref"].append(vnfr_id)
 767
 768             step = "creating nsr at database"
 769             self._format_new_data(session, "nsrs", nsr_descriptor)
 770             self.db.create("nsrs", nsr_descriptor)
 771             rollback.insert(rollback_index, {"item": "nsrs", "_id": nsr_id})
 772             return nsr_id
 773         except Exception as e:
 774             raise EngineException("Error {}: {}".format(step, e))
 775
 776     @staticmethod
 777     def _update_descriptor(desc, kwargs):
 778         """
 779         Update descriptor with the kwargs. It contains dot separated keys
 780         :param desc: dictionary to be updated
 781         :param kwargs: plain dictionary to be used for updating.
 782         :return:
 783         """
 784         if not kwargs:
 785             return
 786         try:
 787             for k, v in kwargs.items():
 788                 update_content = desc
 789                 kitem_old = None
 790                 klist = k.split(".")
 791                 for kitem in klist:
 792                     if kitem_old is not None:
 793                         update_content = update_content[kitem_old]
 794                     if isinstance(update_content, dict):
 795                         kitem_old = kitem
 796                     elif isinstance(update_content, list):
 797                         kitem_old = int(kitem)
 798                     else:
 799                         raise EngineException(
 800                             "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 801                 update_content[kitem_old] = v
 802         except KeyError:
 803             raise EngineException(
 804                 "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 805         except ValueError:
 806             raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 807                 k, kitem))
 808         except IndexError:
 809             raise EngineException(
 810                 "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 811
 812     def new_item(self, rollback, session, item, indata={}, kwargs=None, headers={}, force=False):
 813         """
 814         Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED  entry,
 815         that must be completed with a call to method upload_content
 816         :param rollback: list where this method appends created items at database in case a rollback may to be done
 817         :param session: contains the used login username and working project
 818         :param item: it can be: users, projects, vim_accounts, sdns, nsrs, nsds, vnfds
 819         :param indata: data to be inserted
 820         :param kwargs: used to override the indata descriptor
 821         :param headers: http request headers
 822         :param force: If True avoid some dependence checks
 823         :return: _id: identity of the inserted data.
 824         """
 825
 826         if not session["admin"] and item in ("users", "projects"):
 827             raise EngineException("Needed admin privileges to perform this operation", HTTPStatus.UNAUTHORIZED)
 828
 829         try:
 830             item_envelop = item
 831             if item in ("nsds", "vnfds"):
 832                 item_envelop = "userDefinedData"
 833             content = self._remove_envelop(item_envelop, indata)
 834
 835             # Override descriptor with query string kwargs
 836             self._update_descriptor(content, kwargs)
 837             if not content and item not in ("nsds", "vnfds"):
 838                 raise EngineException("Empty payload")
 839
 840             validate_input(content, item, new=True)
 841
 842             if item == "nsrs":
 843                 # in this case the input descriptor is not the data to be stored
 844                 return self.new_nsr(rollback, session, ns_request=content)
 845
 846             self._validate_new_data(session, item_envelop, content, force=force)
 847             if item in ("nsds", "vnfds"):
 848                 content = {"_admin": {"userDefinedData": content}}
 849             self._format_new_data(session, item, content)
 850             _id = self.db.create(item, content)
 851             rollback.insert(0, {"item": item, "_id": _id})
 852
 853             if item == "vim_accounts":
 854                 msg_data = self.db.get_one(item, {"_id": _id})
 855                 msg_data.pop("_admin", None)
 856                 self.msg.write("vim_account", "create", msg_data)
 857             elif item == "sdns":
 858                 msg_data = self.db.get_one(item, {"_id": _id})
 859                 msg_data.pop("_admin", None)
 860                 self.msg.write("sdn", "create", msg_data)
 861             return _id
 862         except ValidationError as e:
 863             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 864
 865     def new_nslcmop(self, session, nsInstanceId, operation, params):
 866         now = time()
 867         _id = str(uuid4())
 868         nslcmop = {
 869             "id": _id,
 870             "_id": _id,
 871             "operationState": "PROCESSING",  # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
 872             "statusEnteredTime": now,
 873             "nsInstanceId": nsInstanceId,
 874             "lcmOperationType": operation,
 875             "startTime": now,
 876             "isAutomaticInvocation": False,
 877             "operationParams": params,
 878             "isCancelPending": False,
 879             "links": {
 880                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
 881                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsInstanceId,
 882             }
 883         }
 884         return nslcmop
 885
 886     def ns_operation(self, rollback, session, nsInstanceId, operation, indata, kwargs=None):
 887         """
 888         Performs a new operation over a ns
 889         :param rollback: list where this method appends created items at database in case a rollback may to be done
 890         :param session: contains the used login username and working project
 891         :param nsInstanceId: _id of the nsr to perform the operation
 892         :param operation: it can be: instantiate, terminate, action, TODO: update, heal
 893         :param indata: descriptor with the parameters of the operation
 894         :param kwargs: used to override the indata descriptor
 895         :return: id of the nslcmops
 896         """
 897         try:
 898             # Override descriptor with query string kwargs
 899             self._update_descriptor(indata, kwargs)
 900             validate_input(indata, "ns_" + operation, new=True)
 901             # get ns from nsr_id
 902             nsr = self.get_item(session, "nsrs", nsInstanceId)
 903             if not nsr["_admin"].get("nsState") or nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
 904                 if operation == "terminate" and indata.get("autoremove"):
 905                     # NSR must be deleted
 906                     return self.del_item(session, "nsrs", nsInstanceId)
 907                 if operation != "instantiate":
 908                     raise EngineException("ns_instance '{}' cannot be '{}' because it is not instantiated".format(
 909                         nsInstanceId, operation), HTTPStatus.CONFLICT)
 910             else:
 911                 if operation == "instantiate" and not indata.get("force"):
 912                     raise EngineException("ns_instance '{}' cannot be '{}' because it is already instantiated".format(
 913                         nsInstanceId, operation), HTTPStatus.CONFLICT)
 914             indata["nsInstanceId"] = nsInstanceId
 915             self._check_ns_operation(session, nsr, operation, indata)
 916             nslcmop = self.new_nslcmop(session, nsInstanceId, operation, indata)
 917             self._format_new_data(session, "nslcmops", nslcmop)
 918             _id = self.db.create("nslcmops", nslcmop)
 919             rollback.insert(0, {"item": "nslcmops", "_id": _id})
 920             indata["_id"] = _id
 921             self.msg.write("ns", operation, nslcmop)
 922             return _id
 923         except ValidationError as e:
 924             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 925         # except DbException as e:
 926         #     raise EngineException("Cannot get ns_instance '{}': {}".format(e), HTTPStatus.NOT_FOUND)
 927
 928     def _add_read_filter(self, session, item, filter):
 929         if session["admin"]:  # allows all
 930             return filter
 931         if item == "users":
 932             filter["username"] = session["username"]
 933         elif item in ("vnfds", "nsds", "nsrs", "vnfrs"):
 934             filter["_admin.projects_read.cont"] = ["ANY", session["project_id"]]
 935
 936     def _add_delete_filter(self, session, item, filter):
 937         if not session["admin"] and item in ("users", "projects"):
 938             raise EngineException("Only admin users can perform this task", http_code=HTTPStatus.FORBIDDEN)
 939         if item == "users":
 940             if filter.get("_id") == session["username"] or filter.get("username") == session["username"]:
 941                 raise EngineException("You cannot delete your own user", http_code=HTTPStatus.CONFLICT)
 942         elif item == "project":
 943             if filter.get("_id") == session["project_id"]:
 944                 raise EngineException("You cannot delete your own project", http_code=HTTPStatus.CONFLICT)
 945         elif item in ("vnfds", "nsds") and not session["admin"]:
 946             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 947
 948     def get_file(self, session, item, _id, path=None, accept_header=None):
 949         """
 950         Return the file content of a vnfd or nsd
 951         :param session: contains the used login username and working project
 952         :param item: it can be vnfds or nsds
 953         :param _id: Identity of the vnfd, ndsd
 954         :param path: artifact path or "$DESCRIPTOR" or None
 955         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 956         :return: opened file or raises an exception
 957         """
 958         accept_text = accept_zip = False
 959         if accept_header:
 960             if 'text/plain' in accept_header or '*/*' in accept_header:
 961                 accept_text = True
 962             if 'application/zip' in accept_header or '*/*' in accept_header:
 963                 accept_zip = True
 964         if not accept_text and not accept_zip:
 965             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 966                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 967
 968         content = self.get_item(session, item, _id)
 969         if content["_admin"]["onboardingState"] != "ONBOARDED":
 970             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 971                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 972                                   http_code=HTTPStatus.CONFLICT)
 973         storage = content["_admin"]["storage"]
 974         if path is not None and path != "$DESCRIPTOR":   # artifacts
 975             if not storage.get('pkg-dir'):
 976                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 977             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 978                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 979                 return folder_content, "text/plain"
 980                 # TODO manage folders in http
 981             else:
 982                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 983                     "application/octet-stream"
 984
 985         # pkgtype   accept  ZIP  TEXT    -> result
 986         # manyfiles         yes  X       -> zip
 987         #                   no   yes     -> error
 988         # onefile           yes  no      -> zip
 989         #                   X    yes     -> text
 990
 991         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 992             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 993         elif storage.get('pkg-dir') and not accept_zip:
 994             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 995                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 996         else:
 997             if not storage.get('zipfile'):
 998                 # TODO generate zipfile if not present
 999                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
1000                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
1001             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip"
1002
1003     def get_item_list(self, session, item, filter={}):
1004         """
1005         Get a list of items
1006         :param session: contains the used login username and working project
1007         :param item: it can be: users, projects, vnfds, nsds, ...
1008         :param filter: filter of data to be applied
1009         :return: The list, it can be empty if no one match the filter.
1010         """
1011         # TODO add admin to filter, validate rights
1012         # TODO transform data for SOL005 URL requests. Transform filtering
1013         # TODO implement "field-type" query string SOL005
1014
1015         self._add_read_filter(session, item, filter)
1016         return self.db.get_list(item, filter)
1017
1018     def get_item(self, session, item, _id):
1019         """
1020         Get complete information on an items
1021         :param session: contains the used login username and working project
1022         :param item: it can be: users, projects, vnfds, nsds,
1023         :param _id: server id of the item
1024         :return: dictionary, raise exception if not found.
1025         """
1026         filter = {"_id": _id}
1027         # TODO add admin to filter, validate rights
1028         # TODO transform data for SOL005 URL requests
1029         self._add_read_filter(session, item, filter)
1030         return self.db.get_one(item, filter)
1031
1032     def del_item_list(self, session, item, filter={}):
1033         """
1034         Delete a list of items
1035         :param session: contains the used login username and working project
1036         :param item: it can be: users, projects, vnfds, nsds, ...
1037         :param filter: filter of data to be applied
1038         :return: The deleted list, it can be empty if no one match the filter.
1039         """
1040         # TODO add admin to filter, validate rights
1041         self._add_read_filter(session, item, filter)
1042         return self.db.del_list(item, filter)
1043
1044     def del_item(self, session, item, _id, force=False):
1045         """
1046         Delete item by its internal id
1047         :param session: contains the used login username and working project
1048         :param item: it can be: users, projects, vnfds, nsds, ...
1049         :param _id: server id of the item
1050         :param force: indicates if deletion must be forced in case of conflict
1051         :return: dictionary with deleted item _id. It raises exception if not found.
1052         """
1053         # TODO add admin to filter, validate rights
1054         # data = self.get_item(item, _id)
1055         filter = {"_id": _id}
1056         self._add_delete_filter(session, item, filter)
1057         if item in ("vnfds", "nsds") and not force:
1058             descriptor = self.get_item(session, item, _id)
1059             descriptor_id = descriptor.get("id")
1060             if descriptor_id:
1061                 self._check_dependencies_on_descriptor(session, item, descriptor_id, _id)
1062         elif item == "projects":
1063             if not force:
1064                 self._check_project_dependencies(_id)
1065
1066         if item == "nsrs":
1067             nsr = self.db.get_one(item, filter)
1068             if nsr["_admin"].get("nsState") == "INSTANTIATED" and not force:
1069                 raise EngineException("nsr '{}' cannot be deleted because it is in 'INSTANTIATED' state. "
1070                                       "Launch 'terminate' operation first; or force deletion".format(_id),
1071                                       http_code=HTTPStatus.CONFLICT)
1072             v = self.db.del_one(item, {"_id": _id})
1073             self.db.del_list("nslcmops", {"nsInstanceId": _id})
1074             self.db.del_list("vnfrs", {"nsr-id-ref": _id})
1075             self.msg.write("ns", "deleted", {"_id": _id})
1076             return v
1077         if item in ("vim_accounts", "sdns") and not force:
1078             self.db.set_one(item, {"_id": _id}, {"_admin.to_delete": True})   # TODO change status
1079             if item == "vim_accounts":
1080                 self.msg.write("vim_account", "delete", {"_id": _id})
1081             elif item == "sdns":
1082                 self.msg.write("sdn", "delete", {"_id": _id})
1083             return {"deleted": 1}  # TODO indicate an offline operation to return 202 ACCEPTED
1084
1085         v = self.db.del_one(item, filter)
1086         if item in ("vnfds", "nsds"):
1087             self.fs.file_delete(_id, ignore_non_exist=True)
1088         if item in ("vim_accounts", "sdns", "vnfds", "nsds"):
1089             self.msg.write(item[:-1], "deleted", {"_id": _id})
1090         return v
1091
1092     def prune(self):
1093         """
1094         Prune database not needed content
1095         :return: None
1096         """
1097         return self.db.del_list("nsrs", {"_admin.to_delete": True})
1098
1099     def create_admin(self):
1100         """
1101         Creates a new user admin/admin into database if database is empty. Useful for initialization
1102         :return: _id identity of the inserted data, or None
1103         """
1104         users = self.db.get_one("users", fail_on_empty=False, fail_on_more=False)
1105         if users:
1106             return None
1107             # raise EngineException("Unauthorized. Database users is not empty", HTTPStatus.UNAUTHORIZED)
1108         indata = {"username": "admin", "password": "admin", "projects": ["admin"]}
1109         fake_session = {"project_id": "admin", "username": "admin"}
1110         self._format_new_data(fake_session, "users", indata)
1111         _id = self.db.create("users", indata)
1112         return _id
1113
1114     def init_db(self, target_version='1.0'):
1115         """
1116         Init database if empty. If not empty it checks that database version is ok.
1117         If empty, it creates a new user admin/admin at 'users' and a new entry at 'version'
1118         :return: None if ok, exception if error or if the version is different.
1119         """
1120         version = self.db.get_one("version", fail_on_empty=False, fail_on_more=False)
1121         if not version:
1122             # create user admin
1123             self.create_admin()
1124             # create database version
1125             version_data = {
1126                 "_id": '1.0',                     # version text
1127                 "version": 1000,                  # version number
1128                 "date": "2018-04-12",             # version date
1129                 "description": "initial design",  # changes in this version
1130                 'status': 'ENABLED'               # ENABLED, DISABLED (migration in process), ERROR,
1131             }
1132             self.db.create("version", version_data)
1133         elif version["_id"] != target_version:
1134             # TODO implement migration process
1135             raise EngineException("Wrong database version '{}'. Expected '{}'".format(
1136                 version["_id"], target_version), HTTPStatus.INTERNAL_SERVER_ERROR)
1137         elif version["status"] != 'ENABLED':
1138             raise EngineException("Wrong database status '{}'".format(
1139                 version["status"]), HTTPStatus.INTERNAL_SERVER_ERROR)
1140         return
1141
1142     def _edit_item(self, session, item, id, content, indata={}, kwargs=None, force=False):
1143         if indata:
1144             indata = self._remove_envelop(item, indata)
1145
1146         # Override descriptor with query string kwargs
1147         if kwargs:
1148             try:
1149                 for k, v in kwargs.items():
1150                     update_content = indata
1151                     kitem_old = None
1152                     klist = k.split(".")
1153                     for kitem in klist:
1154                         if kitem_old is not None:
1155                             update_content = update_content[kitem_old]
1156                         if isinstance(update_content, dict):
1157                             kitem_old = kitem
1158                         elif isinstance(update_content, list):
1159                             kitem_old = int(kitem)
1160                         else:
1161                             raise EngineException(
1162                                 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
1163                     update_content[kitem_old] = v
1164             except KeyError:
1165                 raise EngineException(
1166                     "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
1167             except ValueError:
1168                 raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
1169                     k, kitem))
1170             except IndexError:
1171                 raise EngineException(
1172                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
1173         try:
1174             validate_input(indata, item, new=False)
1175         except ValidationError as e:
1176             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
1177
1178         self._check_edition(session, item, indata, id, force)
1179         deep_update(content, indata)
1180         self._validate_new_data(session, item, content, id=id, force=force)
1181         # self._format_new_data(session, item, content)
1182         self.db.replace(item, id, content)
1183         if item in ("vim_accounts", "sdns"):
1184             indata.pop("_admin", None)
1185             indata["_id"] = id
1186             if item == "vim_accounts":
1187                 self.msg.write("vim_account", "edit", indata)
1188             elif item == "sdns":
1189                 self.msg.write("sdn", "edit", indata)
1190         return id
1191
1192     def edit_item(self, session, item, _id, indata={}, kwargs=None, force=False):
1193         """
1194         Update an existing entry at database
1195         :param session: contains the used login username and working project
1196         :param item: it can be: users, projects, vnfds, nsds, ...
1197         :param _id: identifier to be updated
1198         :param indata: data to be inserted
1199         :param kwargs: used to override the indata descriptor
1200         :param force: If True avoid some dependence checks
1201         :return: dictionary, raise exception if not found.
1202         """
1203         if not session["admin"] and item == "projects":
1204             raise EngineException("Needed admin privileges to perform this operation", HTTPStatus.UNAUTHORIZED)
1205
1206         content = self.get_item(session, item, _id)
1207         return self._edit_item(session, item, _id, content, indata, kwargs, force)