osm_nbi/engine.py

   1 # -*- coding: utf-8 -*-
   2
   3 from osm_common import dbmongo
   4 from osm_common import dbmemory
   5 from osm_common import fslocal
   6 from osm_common import msglocal
   7 from osm_common import msgkafka
   8 import tarfile
   9 import yaml
  10 import json
  11 import logging
  12 from random import choice as random_choice
  13 from uuid import uuid4
  14 from hashlib import sha256, md5
  15 from osm_common.dbbase import DbException
  16 from osm_common.fsbase import FsException
  17 from osm_common.msgbase import MsgException
  18 from http import HTTPStatus
  19 from time import time
  20 from copy import deepcopy, copy
  21 from validation import validate_input, ValidationError
  22
  23 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  24
  25
  26 class EngineException(Exception):
  27
  28     def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
  29         self.http_code = http_code
  30         Exception.__init__(self, message)
  31
  32
  33 def _deep_update(dict_to_change, dict_reference):
  34     """
  35     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
  36     :param dict_to_change:  Ends modified
  37     :param dict_reference: reference
  38     :return: none
  39     """
  40     for k in dict_reference:
  41         if dict_reference[k] is None:   # None->Anything
  42             if k in dict_to_change:
  43                 del dict_to_change[k]
  44         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
  45             dict_to_change[k] = dict_reference[k]
  46         elif k not in dict_to_change:  # Dict->Empty
  47             dict_to_change[k] = deepcopy(dict_reference[k])
  48             _deep_update(dict_to_change[k], dict_reference[k])
  49         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
  50             _deep_update(dict_to_change[k], dict_reference[k])
  51         else:       # Dict->NotDict
  52             dict_to_change[k] = deepcopy(dict_reference[k])
  53             _deep_update(dict_to_change[k], dict_reference[k])
  54
  55
  56 def get_iterable(input):
  57     """
  58     Returns an iterable, in case input is None it just returns an empty tuple
  59     :param input:
  60     :return: iterable
  61     """
  62     if input is None:
  63         return ()
  64     return input
  65
  66
  67 class Engine(object):
  68
  69     def __init__(self):
  70         self.tokens = {}
  71         self.db = None
  72         self.fs = None
  73         self.msg = None
  74         self.config = None
  75         self.logger = logging.getLogger("nbi.engine")
  76
  77     def start(self, config):
  78         """
  79         Connect to database, filesystem storage, and messaging
  80         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  81         :return: None
  82         """
  83         self.config = config
  84         try:
  85             if not self.db:
  86                 if config["database"]["driver"] == "mongo":
  87                     self.db = dbmongo.DbMongo()
  88                     self.db.db_connect(config["database"])
  89                 elif config["database"]["driver"] == "memory":
  90                     self.db = dbmemory.DbMemory()
  91                     self.db.db_connect(config["database"])
  92                 else:
  93                     raise EngineException("Invalid configuration param '{}' at '[database]':'driver'".format(
  94                         config["database"]["driver"]))
  95             if not self.fs:
  96                 if config["storage"]["driver"] == "local":
  97                     self.fs = fslocal.FsLocal()
  98                     self.fs.fs_connect(config["storage"])
  99                 else:
 100                     raise EngineException("Invalid configuration param '{}' at '[storage]':'driver'".format(
 101                         config["storage"]["driver"]))
 102             if not self.msg:
 103                 if config["message"]["driver"] == "local":
 104                     self.msg = msglocal.MsgLocal()
 105                     self.msg.connect(config["message"])
 106                 elif config["message"]["driver"] == "kafka":
 107                     self.msg = msgkafka.MsgKafka()
 108                     self.msg.connect(config["message"])
 109                 else:
 110                     raise EngineException("Invalid configuration param '{}' at '[message]':'driver'".format(
 111                         config["storage"]["driver"]))
 112         except (DbException, FsException, MsgException) as e:
 113             raise EngineException(str(e), http_code=e.http_code)
 114
 115     def stop(self):
 116         try:
 117             if self.db:
 118                 self.db.db_disconnect()
 119             if self.fs:
 120                 self.fs.fs_disconnect()
 121             if self.fs:
 122                 self.fs.fs_disconnect()
 123         except (DbException, FsException, MsgException) as e:
 124             raise EngineException(str(e), http_code=e.http_code)
 125
 126     def authorize(self, token):
 127         try:
 128             if not token:
 129                 raise EngineException("Needed a token or Authorization http header",
 130                                       http_code=HTTPStatus.UNAUTHORIZED)
 131             if token not in self.tokens:
 132                 raise EngineException("Invalid token or Authorization http header",
 133                                       http_code=HTTPStatus.UNAUTHORIZED)
 134             session = self.tokens[token]
 135             now = time()
 136             if session["expires"] < now:
 137                 del self.tokens[token]
 138                 raise EngineException("Expired Token or Authorization http header",
 139                                       http_code=HTTPStatus.UNAUTHORIZED)
 140             return session
 141         except EngineException:
 142             if self.config["global"].get("test.user_not_authorized"):
 143                 return {"id": "fake-token-id-for-test",
 144                         "project_id": self.config["global"].get("test.project_not_authorized", "admin"),
 145                         "username": self.config["global"]["test.user_not_authorized"]}
 146             else:
 147                 raise
 148
 149     def new_token(self, session, indata, remote):
 150         now = time()
 151         user_content = None
 152
 153         # Try using username/password
 154         if indata.get("username"):
 155             user_rows = self.db.get_list("users", {"username": indata.get("username")})
 156             user_content = None
 157             if user_rows:
 158                 user_content = user_rows[0]
 159                 salt = user_content["_admin"]["salt"]
 160                 shadow_password = sha256(indata.get("password", "").encode('utf-8') + salt.encode('utf-8')).hexdigest()
 161                 if shadow_password != user_content["password"]:
 162                     user_content = None
 163             if not user_content:
 164                 raise EngineException("Invalid username/password", http_code=HTTPStatus.UNAUTHORIZED)
 165         elif session:
 166             user_rows = self.db.get_list("users", {"username": session["username"]})
 167             if user_rows:
 168                 user_content = user_rows[0]
 169             else:
 170                 raise EngineException("Invalid token", http_code=HTTPStatus.UNAUTHORIZED)
 171         else:
 172             raise EngineException("Provide credentials: username/password or Authorization Bearer token",
 173                                   http_code=HTTPStatus.UNAUTHORIZED)
 174
 175         token_id = ''.join(random_choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
 176                            for _ in range(0, 32))
 177         if indata.get("project_id"):
 178             project_id = indata.get("project_id")
 179             if project_id not in user_content["projects"]:
 180                 raise EngineException("project {} not allowed for this user".format(project_id),
 181                                       http_code=HTTPStatus.UNAUTHORIZED)
 182         else:
 183             project_id = user_content["projects"][0]
 184         if project_id == "admin":
 185             session_admin = True
 186         else:
 187             project = self.db.get_one("projects", {"_id": project_id})
 188             session_admin = project.get("admin", False)
 189         new_session = {"issued_at": now, "expires": now+3600,
 190                        "_id": token_id, "id": token_id, "project_id": project_id, "username": user_content["username"],
 191                        "remote_port": remote.port, "admin": session_admin}
 192         if remote.name:
 193             new_session["remote_host"] = remote.name
 194         elif remote.ip:
 195             new_session["remote_host"] = remote.ip
 196
 197         self.tokens[token_id] = new_session
 198         return deepcopy(new_session)
 199
 200     def get_token_list(self, session):
 201         token_list = []
 202         for token_id, token_value in self.tokens.items():
 203             if token_value["username"] == session["username"]:
 204                 token_list.append(deepcopy(token_value))
 205         return token_list
 206
 207     def get_token(self, session, token_id):
 208         token_value = self.tokens.get(token_id)
 209         if not token_value:
 210             raise EngineException("token not found", http_code=HTTPStatus.NOT_FOUND)
 211         if token_value["username"] != session["username"] and not session["admin"]:
 212             raise EngineException("needed admin privileges", http_code=HTTPStatus.UNAUTHORIZED)
 213         return token_value
 214
 215     def del_token(self, token_id):
 216         try:
 217             del self.tokens[token_id]
 218             return "token '{}' deleted".format(token_id)
 219         except KeyError:
 220             raise EngineException("Token '{}' not found".format(token_id), http_code=HTTPStatus.NOT_FOUND)
 221
 222     @staticmethod
 223     def _remove_envelop(item, indata=None):
 224         """
 225         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
 226         vnfd or nsd content
 227         :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds
 228         :param indata: Content to be inspected
 229         :return: the useful part of indata
 230         """
 231         clean_indata = indata
 232         if not indata:
 233             return {}
 234         if item == "vnfds":
 235             if clean_indata.get('vnfd:vnfd-catalog'):
 236                 clean_indata = clean_indata['vnfd:vnfd-catalog']
 237             elif clean_indata.get('vnfd-catalog'):
 238                 clean_indata = clean_indata['vnfd-catalog']
 239             if clean_indata.get('vnfd'):
 240                 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 241                     raise EngineException("'vnfd' must be a list only one element")
 242                 clean_indata = clean_indata['vnfd'][0]
 243         elif item == "nsds":
 244             if clean_indata.get('nsd:nsd-catalog'):
 245                 clean_indata = clean_indata['nsd:nsd-catalog']
 246             elif clean_indata.get('nsd-catalog'):
 247                 clean_indata = clean_indata['nsd-catalog']
 248             if clean_indata.get('nsd'):
 249                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 250                     raise EngineException("'nsd' must be a list only one element")
 251                 clean_indata = clean_indata['nsd'][0]
 252         elif item == "userDefinedData":
 253             if "userDefinedData" in indata:
 254                 clean_indata = clean_indata['userDefinedData']
 255         return clean_indata
 256
 257     def _check_dependencies_on_descriptor(self, session, item, descriptor_id):
 258         """
 259         Check that the descriptor to be deleded is not a dependency of others
 260         :param session: client session information
 261         :param item: can be vnfds, nsds
 262         :param descriptor_id: id of descriptor to be deleted
 263         :return: None or raises exception
 264         """
 265         if item == "vnfds":
 266             _filter = {"constituent-vnfd.ANYINDEX.vnfd-id-ref": descriptor_id}
 267             if self.get_item_list(session, "nsds", _filter):
 268                 raise EngineException("There are nsd that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 269         elif item == "nsds":
 270             _filter = {"nsdId": descriptor_id}
 271             if self.get_item_list(session, "nsrs", _filter):
 272                 raise EngineException("There are nsr that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 273
 274     def _check_descriptor_dependencies(self, session, item, descriptor):
 275         """
 276         Check that the dependent descriptors exist on a new descriptor or edition
 277         :param session: client session information
 278         :param item: can be nsds, nsrs
 279         :param descriptor: descriptor to be inserted or edit
 280         :return: None or raises exception
 281         """
 282         if item == "nsds":
 283             if not descriptor.get("constituent-vnfd"):
 284                 return
 285             for vnf in descriptor["constituent-vnfd"]:
 286                 vnfd_id = vnf["vnfd-id-ref"]
 287                 if not self.get_item_list(session, "vnfds", {"id": vnfd_id}):
 288                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 289                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 290         elif item == "nsrs":
 291             if not descriptor.get("nsdId"):
 292                 return
 293             nsd_id = descriptor["nsdId"]
 294             if not self.get_item_list(session, "nsds", {"id": nsd_id}):
 295                 raise EngineException("Descriptor error at nsdId='{}' references a non exist nsd".format(nsd_id),
 296                                       http_code=HTTPStatus.CONFLICT)
 297
 298     def _validate_new_data(self, session, item, indata, id=None, force=False):
 299         if item == "users":
 300             if not indata.get("username"):
 301                 raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY)
 302             if not indata.get("password"):
 303                 raise EngineException("missing 'password'", HTTPStatus.UNPROCESSABLE_ENTITY)
 304             if not indata.get("projects"):
 305                 raise EngineException("missing 'projects'", HTTPStatus.UNPROCESSABLE_ENTITY)
 306             # check username not exists
 307             if self.db.get_one(item, {"username": indata.get("username")}, fail_on_empty=False, fail_on_more=False):
 308                 raise EngineException("username '{}' exists".format(indata["username"]), HTTPStatus.CONFLICT)
 309         elif item == "projects":
 310             if not indata.get("name"):
 311                 raise EngineException("missing 'name'")
 312             # check name not exists
 313             if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
 314                 raise EngineException("name '{}' exists".format(indata["name"]), HTTPStatus.CONFLICT)
 315         elif item in ("vnfds", "nsds"):
 316             filter = {"id": indata["id"]}
 317             if id:
 318                 filter["_id.neq"] = id
 319             # TODO add admin to filter, validate rights
 320             self._add_read_filter(session, item, filter)
 321             if self.db.get_one(item, filter, fail_on_empty=False):
 322                 raise EngineException("{} with id '{}' already exists for this tenant".format(item[:-1], indata["id"]),
 323                                       HTTPStatus.CONFLICT)
 324             # TODO validate with pyangbind. Load and dumps to convert data types
 325             if item == "nsds":
 326                 # transform constituent-vnfd:member-vnf-index to string
 327                 if indata.get("constituent-vnfd"):
 328                     for constituent_vnfd in indata["constituent-vnfd"]:
 329                         if "member-vnf-index" in constituent_vnfd:
 330                             constituent_vnfd["member-vnf-index"] = str(constituent_vnfd["member-vnf-index"])
 331
 332             if item == "nsds" and not force:
 333                 self._check_descriptor_dependencies(session, "nsds", indata)
 334         elif item == "userDefinedData":
 335             # TODO validate userDefinedData is a keypair values
 336             pass
 337
 338         elif item == "nsrs":
 339             pass
 340         elif item == "vim_accounts" or item == "sdns":
 341             filter = {"name": indata.get("name")}
 342             if id:
 343                 filter["_id.neq"] = id
 344             if self.db.get_one(item, filter, fail_on_empty=False, fail_on_more=False):
 345                 raise EngineException("name '{}' already exists for {}".format(indata["name"], item),
 346                                       HTTPStatus.CONFLICT)
 347
 348     def _check_ns_operation(self, session, nsr, operation, indata):
 349         """
 350         Check that user has enter right parameters for the operation
 351         :param session:
 352         :param operation: it can be: instantiate, terminate, action, TODO: update, heal
 353         :param indata: descriptor with the parameters of the operation
 354         :return: None
 355         """
 356         vnfds = {}
 357         vim_accounts = []
 358         nsd = nsr["nsd"]
 359
 360         def check_valid_vnf_member_index(member_vnf_index):
 361             for vnf in nsd["constituent-vnfd"]:
 362                 if member_vnf_index == vnf["member-vnf-index"]:
 363                     vnfd_id = vnf["vnfd-id-ref"]
 364                     if vnfd_id not in vnfds:
 365                         vnfds[vnfd_id] = self.db.get_one("vnfds", {"id": vnfd_id})
 366                     return vnfds[vnfd_id]
 367             else:
 368                 raise EngineException("Invalid parameter member_vnf_index='{}' is not one of the "
 369                                       "nsd:constituent-vnfd".format(member_vnf_index))
 370
 371         def check_valid_vim_account(vim_account):
 372             if vim_account in vim_accounts:
 373                 return
 374             try:
 375                 self.db.get_one("vim_accounts", {"_id": vim_account})
 376             except Exception:
 377                 raise EngineException("Invalid vimAccountId='{}' not present".format(vim_account))
 378             vim_accounts.append(vim_account)
 379
 380         if operation == "action":
 381             # check vnf_member_index
 382             if indata.get("vnf_member_index"):
 383                 indata["member_vnf_index"] = indata.pop("vnf_member_index")    # for backward compatibility
 384             if not indata.get("member_vnf_index"):
 385                 raise EngineException("Missing 'member_vnf_index' parameter")
 386             vnfd = check_valid_vnf_member_index(indata["member_vnf_index"])
 387             # check primitive
 388             for config_primitive in get_iterable(vnfd.get("vnf-configuration", {}).get("config-primitive")):
 389                 if indata["primitive"] == config_primitive["name"]:
 390                     # check needed primitive_params are provided
 391                     if indata.get("primitive_params"):
 392                         in_primitive_params_copy = copy(indata["primitive_params"])
 393                     else:
 394                         in_primitive_params_copy = {}
 395                     for paramd in get_iterable(config_primitive.get("parameter")):
 396                         if paramd["name"] in in_primitive_params_copy:
 397                             del in_primitive_params_copy[paramd["name"]]
 398                         elif not paramd.get("default-value"):
 399                             raise EngineException("Needed parameter {} not provided for primitive '{}'".format(
 400                                 paramd["name"], indata["primitive"]))
 401                     # check no extra primitive params are provided
 402                     if in_primitive_params_copy:
 403                         raise EngineException("parameter/s '{}' not present at vnfd for primitive '{}'".format(
 404                             list(in_primitive_params_copy.keys()), indata["primitive"]))
 405                     break
 406             else:
 407                 raise EngineException("Invalid primitive '{}' is not present at vnfd".format(indata["primitive"]))
 408         if operation == "scale":
 409             vnfd = check_valid_vnf_member_index(indata["scaleVnfData"]["scaleByStepData"]["member-vnf-index"])
 410             for scaling_group in get_iterable(vnfd.get("scaling-group-descriptor")):
 411                 if indata["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"] == scaling_group["name"]:
 412                     break
 413             else:
 414                 raise EngineException("Invalid scaleVnfData:scaleByStepData:scaling-group-descriptor '{}' is not "
 415                                       "present at vnfd:scaling-group-descriptor".format(
 416                                           indata["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]))
 417         if operation == "instantiate":
 418             # check vim_account
 419             check_valid_vim_account(indata["vimAccountId"])
 420             for in_vnf in get_iterable(indata.get("vnf")):
 421                 vnfd = check_valid_vnf_member_index(in_vnf["member-vnf-index"])
 422                 if in_vnf.get("vimAccountId"):
 423                     check_valid_vim_account(in_vnf["vimAccountId"])
 424                 for in_vdu in get_iterable(in_vnf.get("vdu")):
 425                     for vdud in get_iterable(vnfd.get("vdu")):
 426                         if vdud["id"] == in_vdu["id"]:
 427                             for volume in get_iterable(in_vdu.get("volume")):
 428                                 for volumed in get_iterable(vdud.get("volumes")):
 429                                     if volumed["name"] == volume["name"]:
 430                                         break
 431                                 else:
 432                                     raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:vdu[id='{}']:"
 433                                                           "volume:name='{}' is not present at vnfd:vdu:volumes list".
 434                                                           format(in_vnf["member-vnf-index"], in_vdu["id"],
 435                                                                  volume["name"]))
 436                             break
 437                     else:
 438                         raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:vdu:id='{}' is not "
 439                                               "present at vnfd".format(in_vnf["member-vnf-index"], in_vdu["id"]))
 440
 441                 for in_internal_vld in get_iterable(in_vnf.get("internal-vld")):
 442                     for internal_vldd in get_iterable(vnfd.get("internal-vld")):
 443                         if in_internal_vld["name"] == internal_vldd["name"] or \
 444                                 in_internal_vld["name"] == internal_vldd["id"]:
 445                             break
 446                     else:
 447                         raise EngineException("Invalid parameter vnf[member-vnf-index='{}']:internal-vld:name='{}'"
 448                                               " is not present at vnfd '{}'".format(in_vnf["member-vnf-index"],
 449                                                                                     in_internal_vld["name"],
 450                                                                                     vnfd["id"]))
 451             for in_vld in get_iterable(indata.get("vld")):
 452                 for vldd in get_iterable(nsd.get("vld")):
 453                     if in_vld["name"] == vldd["name"] or in_vld["name"] == vldd["id"]:
 454                             break
 455                     else:
 456                         raise EngineException("Invalid parameter vld:name='{}' is not present at nsd:vld".format(
 457                             in_vld["name"]))
 458
 459     def _format_new_data(self, session, item, indata):
 460         now = time()
 461         if "_admin" not in indata:
 462             indata["_admin"] = {}
 463         indata["_admin"]["created"] = now
 464         indata["_admin"]["modified"] = now
 465         if item == "users":
 466             indata["_id"] = indata["username"]
 467             salt = uuid4().hex
 468             indata["_admin"]["salt"] = salt
 469             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
 470         elif item == "projects":
 471             indata["_id"] = indata["name"]
 472         else:
 473             if not indata.get("_id"):
 474                 indata["_id"] = str(uuid4())
 475             if item in ("vnfds", "nsds", "nsrs", "vnfrs"):
 476                 if not indata["_admin"].get("projects_read"):
 477                     indata["_admin"]["projects_read"] = [session["project_id"]]
 478                 if not indata["_admin"].get("projects_write"):
 479                     indata["_admin"]["projects_write"] = [session["project_id"]]
 480             if item in ("vnfds", "nsds"):
 481                 indata["_admin"]["onboardingState"] = "CREATED"
 482                 indata["_admin"]["operationalState"] = "DISABLED"
 483                 indata["_admin"]["usageSate"] = "NOT_IN_USE"
 484             if item == "nsrs":
 485                 indata["_admin"]["nsState"] = "NOT_INSTANTIATED"
 486             if item in ("vim_accounts", "sdns"):
 487                 indata["_admin"]["operationalState"] = "PROCESSING"
 488
 489     def upload_content(self, session, item, _id, indata, kwargs, headers):
 490         """
 491         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 492         :param session: session
 493         :param item: can be nsds or vnfds
 494         :param _id : the nsd,vnfd is already created, this is the id
 495         :param indata: http body request
 496         :param kwargs: user query string to override parameters. NOT USED
 497         :param headers:  http request headers
 498         :return: True package has is completely uploaded or False if partial content has been uplodaed.
 499             Raise exception on error
 500         """
 501         # Check that _id exists and it is valid
 502         current_desc = self.get_item(session, item, _id)
 503
 504         content_range_text = headers.get("Content-Range")
 505         expected_md5 = headers.get("Content-File-MD5")
 506         compressed = None
 507         content_type = headers.get("Content-Type")
 508         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 509                 "application/zip" in content_type:
 510             compressed = "gzip"
 511         filename = headers.get("Content-Filename")
 512         if not filename:
 513             filename = "package.tar.gz" if compressed else "package"
 514         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 515         file_pkg = None
 516         error_text = ""
 517         try:
 518             if content_range_text:
 519                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 520                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 521                     raise IndexError()
 522                 start = int(content_range[1])
 523                 end = int(content_range[2]) + 1
 524                 total = int(content_range[3])
 525             else:
 526                 start = 0
 527
 528             if start:
 529                 if not self.fs.file_exists(_id, 'dir'):
 530                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 531             else:
 532                 self.fs.file_delete(_id, ignore_non_exist=True)
 533                 self.fs.mkdir(_id)
 534
 535             storage = self.fs.get_params()
 536             storage["folder"] = _id
 537
 538             file_path = (_id, filename)
 539             if self.fs.file_exists(file_path, 'file'):
 540                 file_size = self.fs.file_size(file_path)
 541             else:
 542                 file_size = 0
 543             if file_size != start:
 544                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 545                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 546             file_pkg = self.fs.file_open(file_path, 'a+b')
 547             if isinstance(indata, dict):
 548                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 549                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 550             else:
 551                 indata_len = 0
 552                 while True:
 553                     indata_text = indata.read(4096)
 554                     indata_len += len(indata_text)
 555                     if not indata_text:
 556                         break
 557                     file_pkg.write(indata_text)
 558             if content_range_text:
 559                 if indata_len != end-start:
 560                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 561                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 562                 if end != total:
 563                     # TODO update to UPLOADING
 564                     return False
 565
 566             # PACKAGE UPLOADED
 567             if expected_md5:
 568                 file_pkg.seek(0, 0)
 569                 file_md5 = md5()
 570                 chunk_data = file_pkg.read(1024)
 571                 while chunk_data:
 572                     file_md5.update(chunk_data)
 573                     chunk_data = file_pkg.read(1024)
 574                 if expected_md5 != file_md5.hexdigest():
 575                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 576             file_pkg.seek(0, 0)
 577             if compressed == "gzip":
 578                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 579                 descriptor_file_name = None
 580                 for tarinfo in tar:
 581                     tarname = tarinfo.name
 582                     tarname_path = tarname.split("/")
 583                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 584                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 585                     if len(tarname_path) == 1 and not tarinfo.isdir():
 586                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 587                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 588                         storage["pkg-dir"] = tarname_path[0]
 589                         if len(tarname_path) == 2:
 590                             if descriptor_file_name:
 591                                 raise EngineException(
 592                                     "Found more than one descriptor file at package descriptor tar.gz")
 593                             descriptor_file_name = tarname
 594                 if not descriptor_file_name:
 595                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 596                 storage["descriptor"] = descriptor_file_name
 597                 storage["zipfile"] = filename
 598                 self.fs.file_extract(tar, _id)
 599                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 600                     content = descriptor_file.read()
 601             else:
 602                 content = file_pkg.read()
 603                 storage["descriptor"] = descriptor_file_name = filename
 604
 605             if descriptor_file_name.endswith(".json"):
 606                 error_text = "Invalid json format "
 607                 indata = json.load(content)
 608             else:
 609                 error_text = "Invalid yaml format "
 610                 indata = yaml.load(content)
 611
 612             current_desc["_admin"]["storage"] = storage
 613             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 614             current_desc["_admin"]["operationalState"] = "ENABLED"
 615
 616             self._edit_item(session, item, _id, current_desc, indata, kwargs)
 617             # TODO if descriptor has changed because kwargs update content and remove cached zip
 618             # TODO if zip is not present creates one
 619             return True
 620
 621         except EngineException:
 622             raise
 623         except IndexError:
 624             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 625                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 626         except IOError as e:
 627             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 628         except tarfile.ReadError as e:
 629             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 630         except (ValueError, yaml.YAMLError) as e:
 631             raise EngineException(error_text + str(e))
 632         finally:
 633             if file_pkg:
 634                 file_pkg.close()
 635
 636     def new_nsr(self, rollback, session, ns_request):
 637         """
 638         Creates a new nsr into database. It also creates needed vnfrs
 639         :param rollback: list where this method appends created items at database in case a rollback may to be done
 640         :param session: contains the used login username and working project
 641         :param ns_request: params to be used for the nsr
 642         :return: the _id of nsr descriptor stored at database
 643         """
 644         rollback_index = len(rollback)
 645         step = ""
 646         try:
 647             # look for nsr
 648             step = "getting nsd id='{}' from database".format(ns_request.get("nsdId"))
 649             nsd = self.get_item(session, "nsds", ns_request["nsdId"])
 650             nsr_id = str(uuid4())
 651             now = time()
 652             step = "filling nsr from input data"
 653             nsr_descriptor = {
 654                 "name": ns_request["nsName"],
 655                 "name-ref": ns_request["nsName"],
 656                 "short-name": ns_request["nsName"],
 657                 "admin-status": "ENABLED",
 658                 "nsd": nsd,
 659                 "datacenter": ns_request["vimAccountId"],
 660                 "resource-orchestrator": "osmopenmano",
 661                 "description": ns_request.get("nsDescription", ""),
 662                 "constituent-vnfr-ref": [],
 663
 664                 "operational-status": "init",    # typedef ns-operational-
 665                 "config-status": "init",         # typedef config-states
 666                 "detailed-status": "scheduled",
 667
 668                 "orchestration-progress": {},
 669                 # {"networks": {"active": 0, "total": 0}, "vms": {"active": 0, "total": 0}},
 670
 671                 "crete-time": now,
 672                 "nsd-name-ref": nsd["name"],
 673                 "operational-events": [],   # "id", "timestamp", "description", "event",
 674                 "nsd-ref": nsd["id"],
 675                 "instantiate_params": ns_request,
 676                 "ns-instance-config-ref": nsr_id,
 677                 "id": nsr_id,
 678                 "_id": nsr_id,
 679                 # "input-parameter": xpath, value,
 680                 "ssh-authorized-key": ns_request.get("key-pair-ref"),
 681             }
 682             ns_request["nsr_id"] = nsr_id
 683
 684             # Create VNFR
 685             needed_vnfds = {}
 686             for member_vnf in nsd["constituent-vnfd"]:
 687                 vnfd_id = member_vnf["vnfd-id-ref"]
 688                 step = "getting vnfd id='{}' constituent-vnfd='{}' from database".format(
 689                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 690                 if vnfd_id not in needed_vnfds:
 691                     # Obtain vnfd
 692                     vnf_filter = {"id": vnfd_id}
 693                     self._add_read_filter(session, "vnfds", vnf_filter)
 694                     vnfd = self.db.get_one("vnfds", vnf_filter)
 695                     vnfd.pop("_admin")
 696                     needed_vnfds[vnfd_id] = vnfd
 697                 else:
 698                     vnfd = needed_vnfds[vnfd_id]
 699                 step = "filling vnfr  vnfd-id='{}' constituent-vnfd='{}'".format(
 700                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 701                 vnfr_id = str(uuid4())
 702                 vnfr_descriptor = {
 703                     "id": vnfr_id,
 704                     "_id": vnfr_id,
 705                     "nsr-id-ref": nsr_id,
 706                     "member-vnf-index-ref": member_vnf["member-vnf-index"],
 707                     "created-time": now,
 708                     # "vnfd": vnfd,        # at OSM model.but removed to avoid data duplication TODO: revise
 709                     "vnfd-ref": vnfd_id,
 710                     "vnfd-id": vnfd["_id"],    # not at OSM model, but useful
 711                     "vim-account-id": None,
 712                     "vdur": [],
 713                     "connection-point": [],
 714                     "ip-address": None,  # mgmt-interface filled by LCM
 715                 }
 716                 for cp in vnfd.get("connection-point", ()):
 717                     vnf_cp = {
 718                         "name": cp["name"],
 719                         "connection-point-id": cp.get("id"),
 720                         "id": cp.get("id"),
 721                         # "ip-address", "mac-address" # filled by LCM
 722                         # vim-id  # TODO it would be nice having a vim port id
 723                     }
 724                     vnfr_descriptor["connection-point"].append(vnf_cp)
 725                 for vdu in vnfd["vdu"]:
 726                     vdur_id = str(uuid4())
 727                     vdur = {
 728                         "id": vdur_id,
 729                         "vdu-id-ref": vdu["id"],
 730                         # TODO      "name": ""     Name of the VDU in the VIM
 731                         "ip-address": None,  # mgmt-interface filled by LCM
 732                         # "vim-id", "flavor-id", "image-id", "management-ip" # filled by LCM
 733                         "internal-connection-point": [],
 734                         "interfaces": [],
 735                     }
 736                     # TODO volumes: name, volume-id
 737                     for icp in vdu.get("internal-connection-point", ()):
 738                         vdu_icp = {
 739                             "id": icp["id"],
 740                             "connection-point-id": icp["id"],
 741                             "name": icp.get("name"),
 742                             # "ip-address", "mac-address" # filled by LCM
 743                             # vim-id  # TODO it would be nice having a vim port id
 744                         }
 745                         vdur["internal-connection-point"].append(vdu_icp)
 746                     for iface in vdu.get("interface", ()):
 747                         vdu_iface = {
 748                             "name": iface.get("name"),
 749                             # "ip-address", "mac-address" # filled by LCM
 750                             # vim-id  # TODO it would be nice having a vim port id
 751                         }
 752                         vdur["interfaces"].append(vdu_iface)
 753                     vnfr_descriptor["vdur"].append(vdur)
 754
 755                 step = "creating vnfr vnfd-id='{}' constituent-vnfd='{}' at database".format(
 756                     member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"])
 757                 self._format_new_data(session, "vnfrs", vnfr_descriptor)
 758                 self.db.create("vnfrs", vnfr_descriptor)
 759                 rollback.insert(0, {"item": "vnfrs", "_id": vnfr_id})
 760                 nsr_descriptor["constituent-vnfr-ref"].append(vnfr_id)
 761
 762             step = "creating nsr at database"
 763             self._format_new_data(session, "nsrs", nsr_descriptor)
 764             self.db.create("nsrs", nsr_descriptor)
 765             rollback.insert(rollback_index, {"item": "nsrs", "_id": nsr_id})
 766             return nsr_id
 767         except Exception as e:
 768             raise EngineException("Error {}: {}".format(step, e))
 769
 770     @staticmethod
 771     def _update_descriptor(desc, kwargs):
 772         """
 773         Update descriptor with the kwargs. It contains dot separated keys
 774         :param desc: dictionary to be updated
 775         :param kwargs: plain dictionary to be used for updating.
 776         :return:
 777         """
 778         if not kwargs:
 779             return
 780         try:
 781             for k, v in kwargs.items():
 782                 update_content = desc
 783                 kitem_old = None
 784                 klist = k.split(".")
 785                 for kitem in klist:
 786                     if kitem_old is not None:
 787                         update_content = update_content[kitem_old]
 788                     if isinstance(update_content, dict):
 789                         kitem_old = kitem
 790                     elif isinstance(update_content, list):
 791                         kitem_old = int(kitem)
 792                     else:
 793                         raise EngineException(
 794                             "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 795                 update_content[kitem_old] = v
 796         except KeyError:
 797             raise EngineException(
 798                 "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 799         except ValueError:
 800             raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 801                 k, kitem))
 802         except IndexError:
 803             raise EngineException(
 804                 "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 805
 806     def new_item(self, rollback, session, item, indata={}, kwargs=None, headers={}, force=False):
 807         """
 808         Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED  entry,
 809         that must be completed with a call to method upload_content
 810         :param rollback: list where this method appends created items at database in case a rollback may to be done
 811         :param session: contains the used login username and working project
 812         :param item: it can be: users, projects, vim_accounts, sdns, nsrs, nsds, vnfds
 813         :param indata: data to be inserted
 814         :param kwargs: used to override the indata descriptor
 815         :param headers: http request headers
 816         :param force: If True avoid some dependence checks
 817         :return: _id: identity of the inserted data.
 818         """
 819
 820         try:
 821             item_envelop = item
 822             if item in ("nsds", "vnfds"):
 823                 item_envelop = "userDefinedData"
 824             content = self._remove_envelop(item_envelop, indata)
 825
 826             # Override descriptor with query string kwargs
 827             self._update_descriptor(content, kwargs)
 828             if not indata and item not in ("nsds", "vnfds"):
 829                 raise EngineException("Empty payload")
 830
 831             validate_input(content, item, new=True)
 832
 833             if item == "nsrs":
 834                 # in this case the input descriptor is not the data to be stored
 835                 return self.new_nsr(rollback, session, ns_request=content)
 836
 837             self._validate_new_data(session, item_envelop, content, force)
 838             if item in ("nsds", "vnfds"):
 839                 content = {"_admin": {"userDefinedData": content}}
 840             self._format_new_data(session, item, content)
 841             _id = self.db.create(item, content)
 842             rollback.insert(0, {"item": item, "_id": _id})
 843
 844             if item == "vim_accounts":
 845                 msg_data = self.db.get_one(item, {"_id": _id})
 846                 msg_data.pop("_admin", None)
 847                 self.msg.write("vim_account", "create", msg_data)
 848             elif item == "sdns":
 849                 msg_data = self.db.get_one(item, {"_id": _id})
 850                 msg_data.pop("_admin", None)
 851                 self.msg.write("sdn", "create", msg_data)
 852             return _id
 853         except ValidationError as e:
 854             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 855
 856     def new_nslcmop(self, session, nsInstanceId, operation, params):
 857         now = time()
 858         _id = str(uuid4())
 859         nslcmop = {
 860             "id": _id,
 861             "_id": _id,
 862             "operationState": "PROCESSING",  # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
 863             "statusEnteredTime": now,
 864             "nsInstanceId": nsInstanceId,
 865             "lcmOperationType": operation,
 866             "startTime": now,
 867             "isAutomaticInvocation": False,
 868             "operationParams": params,
 869             "isCancelPending": False,
 870             "links": {
 871                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
 872                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsInstanceId,
 873             }
 874         }
 875         return nslcmop
 876
 877     def ns_operation(self, rollback, session, nsInstanceId, operation, indata, kwargs=None):
 878         """
 879         Performs a new operation over a ns
 880         :param rollback: list where this method appends created items at database in case a rollback may to be done
 881         :param session: contains the used login username and working project
 882         :param nsInstanceId: _id of the nsr to perform the operation
 883         :param operation: it can be: instantiate, terminate, action, TODO: update, heal
 884         :param indata: descriptor with the parameters of the operation
 885         :param kwargs: used to override the indata descriptor
 886         :return: id of the nslcmops
 887         """
 888         try:
 889             # Override descriptor with query string kwargs
 890             self._update_descriptor(indata, kwargs)
 891             validate_input(indata, "ns_" + operation, new=True)
 892             # get ns from nsr_id
 893             nsr = self.get_item(session, "nsrs", nsInstanceId)
 894             if not nsr["_admin"].get("nsState") or nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
 895                 if operation == "terminate" and indata.get("autoremove"):
 896                     # NSR must be deleted
 897                     return self.del_item(session, "nsrs", nsInstanceId)
 898                 if operation != "instantiate":
 899                     raise EngineException("ns_instance '{}' cannot be '{}' because it is not instantiated".format(
 900                         nsInstanceId, operation), HTTPStatus.CONFLICT)
 901             else:
 902                 if operation == "instantiate" and not indata.get("force"):
 903                     raise EngineException("ns_instance '{}' cannot be '{}' because it is already instantiated".format(
 904                         nsInstanceId, operation), HTTPStatus.CONFLICT)
 905             indata["nsInstanceId"] = nsInstanceId
 906             self._check_ns_operation(session, nsr, operation, indata)
 907             nslcmop = self.new_nslcmop(session, nsInstanceId, operation, indata)
 908             self._format_new_data(session, "nslcmops", nslcmop)
 909             _id = self.db.create("nslcmops", nslcmop)
 910             rollback.insert(0, {"item": "nslcmops", "_id": _id})
 911             indata["_id"] = _id
 912             self.msg.write("ns", operation, nslcmop)
 913             return _id
 914         except ValidationError as e:
 915             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 916         # except DbException as e:
 917         #     raise EngineException("Cannot get ns_instance '{}': {}".format(e), HTTPStatus.NOT_FOUND)
 918
 919     def _add_read_filter(self, session, item, filter):
 920         if session["project_id"] == "admin":  # allows all
 921             return filter
 922         if item == "users":
 923             filter["username"] = session["username"]
 924         elif item in ("vnfds", "nsds", "nsrs"):
 925             filter["_admin.projects_read.cont"] = ["ANY", session["project_id"]]
 926
 927     def _add_delete_filter(self, session, item, filter):
 928         if session["project_id"] != "admin" and item in ("users", "projects"):
 929             raise EngineException("Only admin users can perform this task", http_code=HTTPStatus.FORBIDDEN)
 930         if item == "users":
 931             if filter.get("_id") == session["username"] or filter.get("username") == session["username"]:
 932                 raise EngineException("You cannot delete your own user", http_code=HTTPStatus.CONFLICT)
 933         elif item == "project":
 934             if filter.get("_id") == session["project_id"]:
 935                 raise EngineException("You cannot delete your own project", http_code=HTTPStatus.CONFLICT)
 936         elif item in ("vnfds", "nsds") and session["project_id"] != "admin":
 937             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 938
 939     def get_file(self, session, item, _id, path=None, accept_header=None):
 940         """
 941         Return the file content of a vnfd or nsd
 942         :param session: contains the used login username and working project
 943         :param item: it can be vnfds or nsds
 944         :param _id: Identity of the vnfd, ndsd
 945         :param path: artifact path or "$DESCRIPTOR" or None
 946         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 947         :return: opened file or raises an exception
 948         """
 949         accept_text = accept_zip = False
 950         if accept_header:
 951             if 'text/plain' in accept_header or '*/*' in accept_header:
 952                 accept_text = True
 953             if 'application/zip' in accept_header or '*/*' in accept_header:
 954                 accept_zip = True
 955         if not accept_text and not accept_zip:
 956             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 957                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 958
 959         content = self.get_item(session, item, _id)
 960         if content["_admin"]["onboardingState"] != "ONBOARDED":
 961             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 962                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 963                                   http_code=HTTPStatus.CONFLICT)
 964         storage = content["_admin"]["storage"]
 965         if path is not None and path != "$DESCRIPTOR":   # artifacts
 966             if not storage.get('pkg-dir'):
 967                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 968             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 969                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 970                 return folder_content, "text/plain"
 971                 # TODO manage folders in http
 972             else:
 973                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 974                     "application/octet-stream"
 975
 976         # pkgtype   accept  ZIP  TEXT    -> result
 977         # manyfiles         yes  X       -> zip
 978         #                   no   yes     -> error
 979         # onefile           yes  no      -> zip
 980         #                   X    yes     -> text
 981
 982         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 983             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 984         elif storage.get('pkg-dir') and not accept_zip:
 985             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 986                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 987         else:
 988             if not storage.get('zipfile'):
 989                 # TODO generate zipfile if not present
 990                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 991                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 992             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip"
 993
 994     def get_item_list(self, session, item, filter={}):
 995         """
 996         Get a list of items
 997         :param session: contains the used login username and working project
 998         :param item: it can be: users, projects, vnfds, nsds, ...
 999         :param filter: filter of data to be applied
1000         :return: The list, it can be empty if no one match the filter.
1001         """
1002         # TODO add admin to filter, validate rights
1003         # TODO transform data for SOL005 URL requests. Transform filtering
1004         # TODO implement "field-type" query string SOL005
1005
1006         self._add_read_filter(session, item, filter)
1007         return self.db.get_list(item, filter)
1008
1009     def get_item(self, session, item, _id):
1010         """
1011         Get complete information on an items
1012         :param session: contains the used login username and working project
1013         :param item: it can be: users, projects, vnfds, nsds,
1014         :param _id: server id of the item
1015         :return: dictionary, raise exception if not found.
1016         """
1017         filter = {"_id": _id}
1018         # TODO add admin to filter, validate rights
1019         # TODO transform data for SOL005 URL requests
1020         self._add_read_filter(session, item, filter)
1021         return self.db.get_one(item, filter)
1022
1023     def del_item_list(self, session, item, filter={}):
1024         """
1025         Delete a list of items
1026         :param session: contains the used login username and working project
1027         :param item: it can be: users, projects, vnfds, nsds, ...
1028         :param filter: filter of data to be applied
1029         :return: The deleted list, it can be empty if no one match the filter.
1030         """
1031         # TODO add admin to filter, validate rights
1032         self._add_read_filter(session, item, filter)
1033         return self.db.del_list(item, filter)
1034
1035     def del_item(self, session, item, _id, force=False):
1036         """
1037         Delete item by its internal id
1038         :param session: contains the used login username and working project
1039         :param item: it can be: users, projects, vnfds, nsds, ...
1040         :param _id: server id of the item
1041         :param force: indicates if deletion must be forced in case of conflict
1042         :return: dictionary with deleted item _id. It raises exception if not found.
1043         """
1044         # TODO add admin to filter, validate rights
1045         # data = self.get_item(item, _id)
1046         filter = {"_id": _id}
1047         self._add_delete_filter(session, item, filter)
1048         if item in ("vnfds", "nsds") and not force:
1049             descriptor = self.get_item(session, item, _id)
1050             descriptor_id = descriptor.get("id")
1051             if descriptor_id:
1052                 self._check_dependencies_on_descriptor(session, item, descriptor_id)
1053
1054         if item == "nsrs":
1055             nsr = self.db.get_one(item, filter)
1056             if nsr["_admin"].get("nsState") == "INSTANTIATED" and not force:
1057                 raise EngineException("nsr '{}' cannot be deleted because it is in 'INSTANTIATED' state. "
1058                                       "Launch 'terminate' operation first; or force deletion".format(_id),
1059                                       http_code=HTTPStatus.CONFLICT)
1060             v = self.db.del_one(item, {"_id": _id})
1061             self.db.del_list("nslcmops", {"nsInstanceId": _id})
1062             self.db.del_list("vnfrs", {"nsr-id-ref": _id})
1063             self.msg.write("ns", "deleted", {"_id": _id})
1064             return v
1065         if item in ("vim_accounts", "sdns") and not force:
1066             self.db.set_one(item, {"_id": _id}, {"_admin.to_delete": True})   # TODO change status
1067             if item == "vim_accounts":
1068                 self.msg.write("vim_account", "delete", {"_id": _id})
1069             elif item == "sdns":
1070                 self.msg.write("sdn", "delete", {"_id": _id})
1071             return {"deleted": 1}  # TODO indicate an offline operation to return 202 ACCEPTED
1072
1073         v = self.db.del_one(item, filter)
1074         if item in ("vnfds", "nsds"):
1075             self.fs.file_delete(_id, ignore_non_exist=True)
1076         if item in ("vim_accounts", "sdns", "vnfds", "nsds"):
1077             self.msg.write(item[:-1], "deleted", {"_id": _id})
1078         return v
1079
1080     def prune(self):
1081         """
1082         Prune database not needed content
1083         :return: None
1084         """
1085         return self.db.del_list("nsrs", {"_admin.to_delete": True})
1086
1087     def create_admin(self):
1088         """
1089         Creates a new user admin/admin into database if database is empty. Useful for initialization
1090         :return: _id identity of the inserted data, or None
1091         """
1092         users = self.db.get_one("users", fail_on_empty=False, fail_on_more=False)
1093         if users:
1094             return None
1095             # raise EngineException("Unauthorized. Database users is not empty", HTTPStatus.UNAUTHORIZED)
1096         indata = {"username": "admin", "password": "admin", "projects": ["admin"]}
1097         fake_session = {"project_id": "admin", "username": "admin"}
1098         self._format_new_data(fake_session, "users", indata)
1099         _id = self.db.create("users", indata)
1100         return _id
1101
1102     def init_db(self, target_version='1.0'):
1103         """
1104         Init database if empty. If not empty it checks that database version is ok.
1105         If empty, it creates a new user admin/admin at 'users' and a new entry at 'version'
1106         :return: None if ok, exception if error or if the version is different.
1107         """
1108         version = self.db.get_one("version", fail_on_empty=False, fail_on_more=False)
1109         if not version:
1110             # create user admin
1111             self.create_admin()
1112             # create database version
1113             version_data = {
1114                 "_id": '1.0',                     # version text
1115                 "version": 1000,                  # version number
1116                 "date": "2018-04-12",             # version date
1117                 "description": "initial design",  # changes in this version
1118                 'status': 'ENABLED'               # ENABLED, DISABLED (migration in process), ERROR,
1119             }
1120             self.db.create("version", version_data)
1121         elif version["_id"] != target_version:
1122             # TODO implement migration process
1123             raise EngineException("Wrong database version '{}'. Expected '{}'".format(
1124                 version["_id"], target_version), HTTPStatus.INTERNAL_SERVER_ERROR)
1125         elif version["status"] != 'ENABLED':
1126             raise EngineException("Wrong database status '{}'".format(
1127                 version["status"]), HTTPStatus.INTERNAL_SERVER_ERROR)
1128         return
1129
1130     def _edit_item(self, session, item, id, content, indata={}, kwargs=None, force=False):
1131         if indata:
1132             indata = self._remove_envelop(item, indata)
1133
1134         # Override descriptor with query string kwargs
1135         if kwargs:
1136             try:
1137                 for k, v in kwargs.items():
1138                     update_content = indata
1139                     kitem_old = None
1140                     klist = k.split(".")
1141                     for kitem in klist:
1142                         if kitem_old is not None:
1143                             update_content = update_content[kitem_old]
1144                         if isinstance(update_content, dict):
1145                             kitem_old = kitem
1146                         elif isinstance(update_content, list):
1147                             kitem_old = int(kitem)
1148                         else:
1149                             raise EngineException(
1150                                 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
1151                     update_content[kitem_old] = v
1152             except KeyError:
1153                 raise EngineException(
1154                     "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
1155             except ValueError:
1156                 raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
1157                     k, kitem))
1158             except IndexError:
1159                 raise EngineException(
1160                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
1161         try:
1162             validate_input(indata, item, new=False)
1163         except ValidationError as e:
1164             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
1165
1166         _deep_update(content, indata)
1167         self._validate_new_data(session, item, content, id, force)
1168         # self._format_new_data(session, item, content)
1169         self.db.replace(item, id, content)
1170         if item in ("vim_accounts", "sdns"):
1171             indata.pop("_admin", None)
1172             indata["_id"] = id
1173             if item == "vim_accounts":
1174                 self.msg.write("vim_account", "edit", indata)
1175             elif item == "sdns":
1176                 self.msg.write("sdn", "edit", indata)
1177         return id
1178
1179     def edit_item(self, session, item, _id, indata={}, kwargs=None, force=False):
1180         """
1181         Update an existing entry at database
1182         :param session: contains the used login username and working project
1183         :param item: it can be: users, projects, vnfds, nsds, ...
1184         :param _id: identifier to be updated
1185         :param indata: data to be inserted
1186         :param kwargs: used to override the indata descriptor
1187         :param force: If True avoid some dependence checks
1188         :return: dictionary, raise exception if not found.
1189         """
1190
1191         content = self.get_item(session, item, _id)
1192         return self._edit_item(session, item, _id, content, indata, kwargs, force)