osm_nbi/engine.py

   1 # -*- coding: utf-8 -*-
   2
   3 import dbmongo
   4 import dbmemory
   5 import fslocal
   6 import msglocal
   7 import msgkafka
   8 import tarfile
   9 import yaml
  10 import json
  11 import logging
  12 from random import choice as random_choice
  13 from uuid import uuid4
  14 from hashlib import sha256, md5
  15 from dbbase import DbException
  16 from fsbase import FsException
  17 from msgbase import MsgException
  18 from http import HTTPStatus
  19 from time import time
  20 from copy import deepcopy
  21
  22 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  23
  24
  25 class EngineException(Exception):
  26
  27     def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
  28         self.http_code = http_code
  29         Exception.__init__(self, message)
  30
  31
  32 def _deep_update(dict_to_change, dict_reference):
  33     """
  34     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
  35     :param dict_to_change:  Ends modified
  36     :param dict_reference: reference
  37     :return: none
  38     """
  39
  40     for k in dict_reference:
  41         if dict_reference[k] is None:   # None->Anything
  42             if k in dict_to_change:
  43                 del dict_to_change[k]
  44         elif not isinstance(dict_reference[k], dict):  #  NotDict->Anything
  45             dict_to_change[k] = dict_reference[k]
  46         elif k not in dict_to_change:  # Dict->Empty
  47             dict_to_change[k] = deepcopy(dict_reference[k])
  48             _deep_update(dict_to_change[k], dict_reference[k])
  49         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
  50             _deep_update(dict_to_change[k], dict_reference[k])
  51         else:       # Dict->NotDict
  52             dict_to_change[k] = deepcopy(dict_reference[k])
  53             _deep_update(dict_to_change[k], dict_reference[k])
  54
  55 class Engine(object):
  56
  57     def __init__(self):
  58         self.tokens = {}
  59         self.db = None
  60         self.fs = None
  61         self.msg = None
  62         self.config = None
  63         self.logger = logging.getLogger("nbi.engine")
  64
  65     def start(self, config):
  66         """
  67         Connect to database, filesystem storage, and messaging
  68         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  69         :return: None
  70         """
  71         self.config = config
  72         try:
  73             if not self.db:
  74                 if config["database"]["driver"] == "mongo":
  75                     self.db = dbmongo.DbMongo()
  76                     self.db.db_connect(config["database"])
  77                 elif config["database"]["driver"] == "memory":
  78                     self.db = dbmemory.DbMemory()
  79                     self.db.db_connect(config["database"])
  80                 else:
  81                     raise EngineException("Invalid configuration param '{}' at '[database]':'driver'".format(
  82                         config["database"]["driver"]))
  83             if not self.fs:
  84                 if config["storage"]["driver"] == "local":
  85                     self.fs = fslocal.FsLocal()
  86                     self.fs.fs_connect(config["storage"])
  87                 else:
  88                     raise EngineException("Invalid configuration param '{}' at '[storage]':'driver'".format(
  89                         config["storage"]["driver"]))
  90             if not self.msg:
  91                 if config["message"]["driver"] == "local":
  92                     self.msg = msglocal.MsgLocal()
  93                     self.msg.connect(config["message"])
  94                 elif config["message"]["driver"] == "kafka":
  95                     self.msg = msgkafka.MsgKafka()
  96                     self.msg.connect(config["message"])
  97                 else:
  98                     raise EngineException("Invalid configuration param '{}' at '[message]':'driver'".format(
  99                         config["storage"]["driver"]))
 100         except (DbException, FsException, MsgException) as e:
 101             raise EngineException(str(e), http_code=e.http_code)
 102
 103     def stop(self):
 104         try:
 105             if self.db:
 106                 self.db.db_disconnect()
 107             if self.fs:
 108                 self.fs.fs_disconnect()
 109             if self.fs:
 110                 self.fs.fs_disconnect()
 111         except (DbException, FsException, MsgException) as e:
 112             raise EngineException(str(e), http_code=e.http_code)
 113
 114     def authorize(self, token):
 115         try:
 116             if not token:
 117                 raise EngineException("Needed a token or Authorization http header",
 118                                       http_code=HTTPStatus.UNAUTHORIZED)
 119             if token not in self.tokens:
 120                 raise EngineException("Invalid token or Authorization http header",
 121                                       http_code=HTTPStatus.UNAUTHORIZED)
 122             session = self.tokens[token]
 123             now = time()
 124             if session["expires"] < now:
 125                 del self.tokens[token]
 126                 raise EngineException("Expired Token or Authorization http header",
 127                                       http_code=HTTPStatus.UNAUTHORIZED)
 128             return session
 129         except EngineException:
 130             if self.config["global"].get("test.user_not_authorized"):
 131                 return {"id": "fake-token-id-for-test",
 132                         "project_id": self.config["global"].get("test.project_not_authorized", "admin"),
 133                         "username": self.config["global"]["test.user_not_authorized"]}
 134             else:
 135                 raise
 136
 137     def new_token(self, session, indata, remote):
 138         now = time()
 139         user_content = None
 140
 141         # Try using username/password
 142         if indata.get("username"):
 143             user_rows = self.db.get_list("users", {"username": indata.get("username")})
 144             user_content = None
 145             if user_rows:
 146                 user_content = user_rows[0]
 147                 salt = user_content["_admin"]["salt"]
 148                 shadow_password = sha256(indata.get("password", "").encode('utf-8') + salt.encode('utf-8')).hexdigest()
 149                 if shadow_password != user_content["password"]:
 150                     user_content = None
 151             if not user_content:
 152                 raise EngineException("Invalid username/password", http_code=HTTPStatus.UNAUTHORIZED)
 153         elif session:
 154             user_rows = self.db.get_list("users", {"username": session["username"]})
 155             if user_rows:
 156                 user_content = user_rows[0]
 157             else:
 158                 raise EngineException("Invalid token", http_code=HTTPStatus.UNAUTHORIZED)
 159         else:
 160             raise EngineException("Provide credentials: username/password or Authorization Bearer token",
 161                                   http_code=HTTPStatus.UNAUTHORIZED)
 162
 163         token_id = ''.join(random_choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
 164                            for _ in range(0, 32))
 165         if indata.get("project_id"):
 166             project_id = indata.get("project_id")
 167             if project_id not in user_content["projects"]:
 168                 raise EngineException("project {} not allowed for this user".format(project_id),
 169                                       http_code=HTTPStatus.UNAUTHORIZED)
 170         else:
 171             project_id = user_content["projects"][0]
 172         if project_id == "admin":
 173             session_admin = True
 174         else:
 175             project = self.db.get_one("projects", {"_id": project_id})
 176             session_admin = project.get("admin", False)
 177         new_session = {"issued_at": now, "expires": now+3600,
 178                        "_id": token_id, "id": token_id, "project_id": project_id, "username": user_content["username"],
 179                        "remote_port": remote.port, "admin": session_admin}
 180         if remote.name:
 181             new_session["remote_host"] = remote.name
 182         elif remote.ip:
 183             new_session["remote_host"] = remote.ip
 184
 185         self.tokens[token_id] = new_session
 186         return deepcopy(new_session)
 187
 188     def get_token_list(self, session):
 189         token_list = []
 190         for token_id, token_value in self.tokens.items():
 191             if token_value["username"] == session["username"]:
 192                 token_list.append(deepcopy(token_value))
 193         return token_list
 194
 195     def get_token(self, session, token_id):
 196         token_value = self.tokens.get(token_id)
 197         if not token_value:
 198             raise EngineException("token not found", http_code=HTTPStatus.NOT_FOUND)
 199         if token_value["username"] != session["username"] and not session["admin"]:
 200             raise EngineException("needed admin privileges", http_code=HTTPStatus.UNAUTHORIZED)
 201         return token_value
 202
 203     def del_token(self, token_id):
 204         try:
 205             del self.tokens[token_id]
 206             return "token '{}' deleted".format(token_id)
 207         except KeyError:
 208             raise EngineException("Token '{}' not found".format(token_id), http_code=HTTPStatus.NOT_FOUND)
 209
 210     @staticmethod
 211     def _remove_envelop(item, indata=None):
 212         """
 213         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
 214         vnfd or nsd content
 215         :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds
 216         :param indata: Content to be inspected
 217         :return: the useful part of indata
 218         """
 219         clean_indata = indata
 220         if not indata:
 221             return {}
 222         if item == "vnfds":
 223             if clean_indata.get('vnfd:vnfd-catalog'):
 224                 clean_indata = clean_indata['vnfd:vnfd-catalog']
 225             elif clean_indata.get('vnfd-catalog'):
 226                 clean_indata = clean_indata['vnfd-catalog']
 227             if clean_indata.get('vnfd'):
 228                 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 229                     raise EngineException("'vnfd' must be a list only one element")
 230                 clean_indata = clean_indata['vnfd'][0]
 231         elif item == "nsds":
 232             if clean_indata.get('nsd:nsd-catalog'):
 233                 clean_indata = clean_indata['nsd:nsd-catalog']
 234             elif clean_indata.get('nsd-catalog'):
 235                 clean_indata = clean_indata['nsd-catalog']
 236             if clean_indata.get('nsd'):
 237                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 238                     raise EngineException("'nsd' must be a list only one element")
 239                 clean_indata = clean_indata['nsd'][0]
 240         elif item == "userDefinedData":
 241             if "userDefinedData" in indata:
 242                 clean_indata = clean_indata['userDefinedData']
 243         return clean_indata
 244
 245     def _validate_new_data(self, session, item, indata, id=None):
 246         if item == "users":
 247             if not indata.get("username"):
 248                 raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY)
 249             if not indata.get("password"):
 250                 raise EngineException("missing 'password'", HTTPStatus.UNPROCESSABLE_ENTITY)
 251             if not indata.get("projects"):
 252                 raise EngineException("missing 'projects'", HTTPStatus.UNPROCESSABLE_ENTITY)
 253             # check username not exist
 254             if self.db.get_one(item, {"username": indata.get("username")}, fail_on_empty=False, fail_on_more=False):
 255                 raise EngineException("username '{}' exist".format(indata["username"]), HTTPStatus.CONFLICT)
 256         elif item == "projects":
 257             if not indata.get("name"):
 258                 raise EngineException("missing 'name'")
 259             # check name not exist
 260             if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
 261                 raise EngineException("name '{}' exist".format(indata["name"]), HTTPStatus.CONFLICT)
 262         elif item in ("vnfds", "nsds"):
 263             filter = {"id": indata["id"]}
 264             if id:
 265                 filter["_id.neq"] = id
 266             # TODO add admin to filter, validate rights
 267             self._add_read_filter(session, item, filter)
 268             if self.db.get_one(item, filter, fail_on_empty=False):
 269                 raise EngineException("{} with id '{}' already exist for this tenant".format(item[:-1], indata["id"]),
 270                                       HTTPStatus.CONFLICT)
 271
 272             # TODO validate with pyangbind
 273         elif item == "userDefinedData":
 274             # TODO validate userDefinedData is a keypair values
 275             pass
 276
 277         elif item == "nsrs":
 278             pass
 279
 280     def _format_new_data(self, session, item, indata, admin=None):
 281         now = time()
 282         if not "_admin" in indata:
 283             indata["_admin"] = {}
 284         indata["_admin"]["created"] = now
 285         indata["_admin"]["modified"] = now
 286         if item == "users":
 287             _id = indata["username"]
 288             salt = uuid4().hex
 289             indata["_admin"]["salt"] = salt
 290             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
 291         elif item == "projects":
 292             _id = indata["name"]
 293         else:
 294             _id = None
 295             storage = None
 296             if admin:
 297                 _id = admin.get("_id")
 298                 storage = admin.get("storage")
 299             if not _id:
 300                 _id = str(uuid4())
 301             if item in ("vnfds", "nsds"):
 302                 if not indata["_admin"].get("projects_read"):
 303                     indata["_admin"]["projects_read"] = [session["project_id"]]
 304                 if not indata["_admin"].get("projects_write"):
 305                     indata["_admin"]["projects_write"] = [session["project_id"]]
 306                 indata["_admin"]["onboardingState"] = "CREATED"
 307                 indata["_admin"]["operationalState"] = "DISABLED"
 308                 indata["_admin"]["usageSate"] = "NOT_IN_USE"
 309             if storage:
 310                 indata["_admin"]["storage"] = storage
 311         indata["_id"] = _id
 312
 313     def upload_content(self, session, item, _id, indata, kwargs, headers):
 314         """
 315         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 316         :param session: session
 317         :param item: can be nsds or vnfds
 318         :param _id : the nsd,vnfd is already created, this is the id
 319         :param indata: http body request
 320         :param kwargs: user query string to override parameters. NOT USED
 321         :param headers:  http request headers
 322         :return: True package has is completely uploaded or False if partial content has been uplodaed.
 323             Raise exception on error
 324         """
 325         # Check that _id exist and it is valid
 326         current_desc = self.get_item(session, item, _id)
 327
 328         content_range_text = headers.get("Content-Range")
 329         expected_md5 = headers.get("Content-File-MD5")
 330         compressed = None
 331         content_type = headers.get("Content-Type")
 332         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 333                 "application/zip" in content_type:
 334             compressed = "gzip"
 335         filename = headers.get("Content-Filename")
 336         if not filename:
 337             filename = "package.tar.gz" if compressed else "package"
 338         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 339         file_pkg = None
 340         error_text = ""
 341         try:
 342             if content_range_text:
 343                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 344                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 345                     raise IndexError()
 346                 start = int(content_range[1])
 347                 end = int(content_range[2]) + 1
 348                 total = int(content_range[3])
 349             else:
 350                 start = 0
 351
 352             if start:
 353                 if not self.fs.file_exists(_id, 'dir'):
 354                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 355             else:
 356                 self.fs.file_delete(_id, ignore_non_exist=True)
 357                 self.fs.mkdir(_id)
 358
 359             storage = self.fs.get_params()
 360             storage["folder"] = _id
 361
 362             file_path = (_id, filename)
 363             if self.fs.file_exists(file_path, 'file'):
 364                 file_size = self.fs.file_size(file_path)
 365             else:
 366                 file_size = 0
 367             if file_size != start:
 368                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 369                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 370             file_pkg = self.fs.file_open(file_path, 'a+b')
 371             if isinstance(indata, dict):
 372                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 373                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 374             else:
 375                 indata_len = 0
 376                 while True:
 377                     indata_text = indata.read(4096)
 378                     indata_len += len(indata_text)
 379                     if not indata_text:
 380                         break
 381                     file_pkg.write(indata_text)
 382             if content_range_text:
 383                 if indata_len != end-start:
 384                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 385                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 386                 if end != total:
 387                     # TODO update to UPLOADING
 388                     return False
 389
 390             # PACKAGE UPLOADED
 391             if expected_md5:
 392                 file_pkg.seek(0, 0)
 393                 file_md5 = md5()
 394                 chunk_data = file_pkg.read(1024)
 395                 while chunk_data:
 396                     file_md5.update(chunk_data)
 397                     chunk_data = file_pkg.read(1024)
 398                 if expected_md5 != file_md5.hexdigest():
 399                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 400             file_pkg.seek(0, 0)
 401             if compressed == "gzip":
 402                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 403                 descriptor_file_name = None
 404                 for tarinfo in tar:
 405                     tarname = tarinfo.name
 406                     tarname_path = tarname.split("/")
 407                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 408                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 409                     if len(tarname_path) == 1 and not tarinfo.isdir():
 410                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 411                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 412                         storage["pkg-dir"] = tarname_path[0]
 413                         if len(tarname_path) == 2:
 414                             if descriptor_file_name:
 415                                 raise EngineException("Found more than one descriptor file at package descriptor tar.gz")
 416                             descriptor_file_name = tarname
 417                 if not descriptor_file_name:
 418                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 419                 storage["descriptor"] = descriptor_file_name
 420                 storage["zipfile"] = filename
 421                 self.fs.file_extract(tar, _id)
 422                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 423                     content = descriptor_file.read()
 424             else:
 425                 content = file_pkg.read()
 426                 storage["descriptor"] = descriptor_file_name = filename
 427
 428             if descriptor_file_name.endswith(".json"):
 429                 error_text = "Invalid json format "
 430                 indata = json.load(content)
 431             else:
 432                 error_text = "Invalid yaml format "
 433                 indata = yaml.load(content)
 434
 435             current_desc["_admin"]["storage"] = storage
 436             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 437             current_desc["_admin"]["operationalState"] = "ENABLED"
 438
 439             self._edit_item(session, item, _id, current_desc, indata, kwargs)
 440             # TODO if descriptor has changed because kwargs update content and remove cached zip
 441             # TODO if zip is not present creates one
 442             return True
 443
 444         except EngineException:
 445             raise
 446         except IndexError:
 447             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 448                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 449         except IOError as e:
 450             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 451         except (ValueError, yaml.YAMLError) as e:
 452             raise EngineException(error_text + str(e))
 453         finally:
 454             if file_pkg:
 455                 file_pkg.close()
 456
 457     def new_nsr(self, session, ns_request):
 458         """
 459         Creates a new nsr into database
 460         :param session: contains the used login username and working project
 461         :param ns_request: params to be used for the nsr
 462         :return: nsr descriptor to be stored at database and the _id
 463         """
 464
 465         # look for nsr
 466         nsd = self.get_item(session, "nsds", ns_request["nsdId"])
 467         _id = str(uuid4())
 468         nsr_descriptor = {
 469             "name": ns_request["nsName"],
 470             "name-ref": ns_request["nsName"],
 471             "short-name": ns_request["nsName"],
 472             "admin-status": "ENABLED",
 473             "nsd": nsd,
 474             "datacenter": ns_request["vimAccountId"],
 475             "resource-orchestrator": "osmopenmano",
 476             "description": ns_request.get("nsDescription", ""),
 477             "constituent-vnfr-ref": ["TODO datacenter-id, vnfr-id"],
 478
 479             "operational-status": "init",    #  typedef ns-operational-
 480             "config-status": "init",         #  typedef config-states
 481             "detailed-status": "scheduled",
 482
 483             "orchestration-progress": {},  # {"networks": {"active": 0, "total": 0}, "vms": {"active": 0, "total": 0}},
 484
 485             "crete-time": time(),
 486             "nsd-name-ref": nsd["name"],
 487             "operational-events": [],   # "id", "timestamp", "description", "event",
 488             "nsd-ref": nsd["id"],
 489             "ns-instance-config-ref": _id,
 490             "id": _id,
 491
 492             # "input-parameter": xpath, value,
 493             "ssh-authorized-key": ns_request.get("key-pair-ref"),
 494         }
 495         ns_request["nsr_id"] = _id
 496         return nsr_descriptor, _id
 497
 498     def new_item(self, session, item, indata={}, kwargs=None, headers={}):
 499         """
 500         Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED  entry,
 501         that must be completed with a call to method upload_content
 502         :param session: contains the used login username and working project
 503         :param item: it can be: users, projects, nsrs, nsds, vnfds
 504         :param indata: data to be inserted
 505         :param kwargs: used to override the indata descriptor
 506         :param headers: http request headers
 507         :return: _id, transaction_id: identity of the inserted data. or transaction_id if Content-Range is used
 508         """
 509         # TODO validate input. Check not exist at database
 510         # TODO add admin and status
 511
 512         transaction = None
 513         # if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \
 514         #     "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type") or \
 515         #     "text/plain" in headers.get("Content-Type"):
 516         #     if not indata:
 517         #         raise EngineException("Empty payload")
 518         #     transaction = self._new_item_partial(session, item, indata, headers)
 519         #     if "desc" not in transaction:
 520         #         return transaction["_id"], False
 521         #     indata = transaction["desc"]
 522
 523         item_envelop = item
 524         if item in ("nsds", "vnfds"):
 525             item_envelop = "userDefinedData"
 526         content = self._remove_envelop(item_envelop, indata)
 527
 528         # Override descriptor with query string kwargs
 529         if kwargs:
 530             try:
 531                 for k, v in kwargs.items():
 532                     update_content = content
 533                     kitem_old = None
 534                     klist = k.split(".")
 535                     for kitem in klist:
 536                         if kitem_old is not None:
 537                             update_content = update_content[kitem_old]
 538                         if isinstance(update_content, dict):
 539                             kitem_old = kitem
 540                         elif isinstance(update_content, list):
 541                             kitem_old = int(kitem)
 542                         else:
 543                             raise EngineException(
 544                                 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 545                     update_content[kitem_old] = v
 546             except KeyError:
 547                 raise EngineException(
 548                     "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 549             except ValueError:
 550                 raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 551                     k, kitem))
 552             except IndexError:
 553                 raise EngineException(
 554                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 555         if not indata and item not in ("nsds", "vnfds"):
 556             raise EngineException("Empty payload")
 557
 558         if item == "nsrs":
 559             # in this case the imput descriptor is not the data to be stored
 560             ns_request = content
 561             content, _id = self.new_nsr(session, ns_request)
 562             transaction = {"_id": _id}
 563
 564         self._validate_new_data(session, item_envelop, content)
 565         if item in ("nsds", "vnfds"):
 566             content = {"_admin": {"userDefinedData": content}}
 567         self._format_new_data(session, item, content, transaction)
 568         _id = self.db.create(item, content)
 569         if item == "nsrs":
 570             self.msg.write("ns", "create", _id)
 571         return _id
 572
 573     def _add_read_filter(self, session, item, filter):
 574         if session["project_id"] == "admin":  # allows all
 575             return filter
 576         if item == "users":
 577             filter["username"] = session["username"]
 578         elif item == "vnfds" or item == "nsds":
 579             filter["_admin.projects_read.cont"] = ["ANY", session["project_id"]]
 580
 581     def _add_delete_filter(self, session, item, filter):
 582         if session["project_id"] != "admin" and item in ("users", "projects"):
 583             raise EngineException("Only admin users can perform this task", http_code=HTTPStatus.FORBIDDEN)
 584         if item == "users":
 585             if filter.get("_id") == session["username"] or filter.get("username") == session["username"]:
 586                 raise EngineException("You cannot delete your own user", http_code=HTTPStatus.CONFLICT)
 587         elif item == "project":
 588             if filter.get("_id") == session["project_id"]:
 589                 raise EngineException("You cannot delete your own project", http_code=HTTPStatus.CONFLICT)
 590         elif item in ("vnfds", "nsds") and session["project_id"] != "admin":
 591             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 592
 593     def get_file(self, session, item, _id, path=None, accept_header=None):
 594         """
 595         Return the file content of a vnfd or nsd
 596         :param session: contains the used login username and working project
 597         :param item: it can be vnfds or nsds
 598         :param _id: Identity of the vnfd, ndsd
 599         :param path: artifact path or "$DESCRIPTOR" or None
 600         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 601         :return: opened file or raises an exception
 602         """
 603         accept_text = accept_zip = False
 604         if accept_header:
 605             if 'text/plain' in accept_header or '*/*' in accept_header:
 606                 accept_text = True
 607             if 'application/zip' in accept_header or '*/*' in accept_header:
 608                 accept_zip = True
 609         if not accept_text and not accept_zip:
 610             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 611                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 612
 613         content = self.get_item(session, item, _id)
 614         if content["_admin"]["onboardingState"] != "ONBOARDED":
 615             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 616                 "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 617                                   http_code=HTTPStatus.CONFLICT)
 618         storage = content["_admin"]["storage"]
 619         if path is not None and path != "$DESCRIPTOR":   # artifacts
 620             if not storage.get('pkg-dir'):
 621                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 622             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 623                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 624                 return folder_content, "text/plain"
 625                 # TODO manage folders in http
 626             else:
 627                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 628                        "application/octet-stream"
 629
 630         # pkgtype   accept  ZIP  TEXT    -> result
 631         # manyfiles         yes  X       -> zip
 632         #                   no   yes     -> error
 633         # onefile           yes  no      -> zip
 634         #                   X    yes     -> text
 635
 636         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 637             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 638         elif storage.get('pkg-dir') and not accept_zip:
 639             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 640                                       "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 641         else:
 642             if not storage.get('zipfile'):
 643                 # TODO generate zipfile if not present
 644                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in future versions"
 645                                       "", http_code=HTTPStatus.NOT_ACCEPTABLE)
 646             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip"
 647
 648
 649     def get_item_list(self, session, item, filter={}):
 650         """
 651         Get a list of items
 652         :param session: contains the used login username and working project
 653         :param item: it can be: users, projects, vnfds, nsds, ...
 654         :param filter: filter of data to be applied
 655         :return: The list, it can be empty if no one match the filter.
 656         """
 657         # TODO add admin to filter, validate rights
 658         # TODO transform data for SOL005 URL requests. Transform filtering
 659         # TODO implement "field-type" query string SOL005
 660
 661         self._add_read_filter(session, item, filter)
 662         return self.db.get_list(item, filter)
 663
 664     def get_item(self, session, item, _id):
 665         """
 666         Get complete information on an items
 667         :param session: contains the used login username and working project
 668         :param item: it can be: users, projects, vnfds, nsds,
 669         :param _id: server id of the item
 670         :return: dictionary, raise exception if not found.
 671         """
 672         database_item = item
 673         filter = {"_id": _id}
 674         # TODO add admin to filter, validate rights
 675         # TODO transform data for SOL005 URL requests
 676         self._add_read_filter(session, item, filter)
 677         return self.db.get_one(item, filter)
 678
 679     def del_item_list(self, session, item, filter={}):
 680         """
 681         Delete a list of items
 682         :param session: contains the used login username and working project
 683         :param item: it can be: users, projects, vnfds, nsds, ...
 684         :param filter: filter of data to be applied
 685         :return: The deleted list, it can be empty if no one match the filter.
 686         """
 687         # TODO add admin to filter, validate rights
 688         self._add_read_filter(session, item, filter)
 689         return self.db.del_list(item, filter)
 690
 691     def del_item(self, session, item, _id):
 692         """
 693         Get complete information on an items
 694         :param session: contains the used login username and working project
 695         :param item: it can be: users, projects, vnfds, nsds, ...
 696         :param _id: server id of the item
 697         :return: dictionary, raise exception if not found.
 698         """
 699         # TODO add admin to filter, validate rights
 700         # data = self.get_item(item, _id)
 701         filter = {"_id": _id}
 702         self._add_delete_filter(session, item, filter)
 703
 704         if item == "nsrs":
 705             desc = self.db.get_one(item, filter)
 706             desc["_admin"]["to_delete"] = True
 707             self.db.replace(item, _id, desc)   # TODO change to set_one
 708             self.msg.write("ns", "delete", _id)
 709             return {"deleted": 1}
 710
 711         v = self.db.del_one(item, filter)
 712         self.fs.file_delete(_id, ignore_non_exist=True)
 713         if item == "nsrs":
 714             self.msg.write("ns", "delete", _id)
 715         return v
 716
 717     def prune(self):
 718         """
 719         Prune database not needed content
 720         :return: None
 721         """
 722         return self.db.del_list("nsrs", {"_admin.to_delete": True})
 723
 724     def create_admin(self):
 725         """
 726         Creates a new user admin/admin into database. Only allowed if database is empty. Useful for initialization
 727         :return: _id identity of the inserted data.
 728         """
 729         users = self.db.get_one("users", fail_on_empty=False, fail_on_more=False)
 730         if users:
 731             raise EngineException("Unauthorized. Database users is not empty", HTTPStatus.UNAUTHORIZED)
 732         indata = {"username": "admin", "password": "admin", "projects": ["admin"]}
 733         fake_session = {"project_id": "admin", "username": "admin"}
 734         self._format_new_data(fake_session, "users", indata)
 735         _id = self.db.create("users", indata)
 736         return _id
 737
 738     def _edit_item(self, session, item, id, content, indata={}, kwargs=None):
 739         if indata:
 740             indata = self._remove_envelop(item, indata)
 741
 742         # Override descriptor with query string kwargs
 743         if kwargs:
 744             try:
 745                 for k, v in kwargs.items():
 746                     update_content = indata
 747                     kitem_old = None
 748                     klist = k.split(".")
 749                     for kitem in klist:
 750                         if kitem_old is not None:
 751                             update_content = update_content[kitem_old]
 752                         if isinstance(update_content, dict):
 753                             kitem_old = kitem
 754                         elif isinstance(update_content, list):
 755                             kitem_old = int(kitem)
 756                         else:
 757                             raise EngineException(
 758                                 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 759                     update_content[kitem_old] = v
 760             except KeyError:
 761                 raise EngineException(
 762                     "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 763             except ValueError:
 764                 raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 765                     k, kitem))
 766             except IndexError:
 767                 raise EngineException(
 768                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 769
 770         _deep_update(content, indata)
 771         self._validate_new_data(session, item, content, id)
 772         # self._format_new_data(session, item, content)
 773         self.db.replace(item, id, content)
 774         return id
 775
 776     def edit_item(self, session, item, _id, indata={}, kwargs=None):
 777         """
 778         Update an existing entry at database
 779         :param session: contains the used login username and working project
 780         :param item: it can be: users, projects, vnfds, nsds, ...
 781         :param _id: identifier to be updated
 782         :param indata: data to be inserted
 783         :param kwargs: used to override the indata descriptor
 784         :return: dictionary, raise exception if not found.
 785         """
 786
 787         content = self.get_item(session, item, _id)
 788         return self._edit_item(session, item, _id, content, indata, kwargs)
 789
 790
 791