osm_nbi/base_topic.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import logging
  17 from uuid import uuid4
  18 from http import HTTPStatus
  19 from time import time
  20 from osm_common.dbbase import deep_update_rfc7396
  21 from osm_nbi.validation import validate_input, ValidationError, is_valid_uuid
  22
  23 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  24
  25
  26 class EngineException(Exception):
  27
  28     def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
  29         self.http_code = http_code
  30         super(Exception, self).__init__(message)
  31
  32
  33 def deep_get(target_dict, key_list):
  34     """
  35     Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
  36     Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
  37     :param target_dict: dictionary to be read
  38     :param key_list: list of keys to read from  target_dict
  39     :return: The wanted value if exist, None otherwise
  40     """
  41     for key in key_list:
  42         if not isinstance(target_dict, dict) or key not in target_dict:
  43             return None
  44         target_dict = target_dict[key]
  45     return target_dict
  46
  47
  48 def get_iterable(input_var):
  49     """
  50     Returns an iterable, in case input_var is None it just returns an empty tuple
  51     :param input_var: can be a list, tuple or None
  52     :return: input_var or () if it is None
  53     """
  54     if input_var is None:
  55         return ()
  56     return input_var
  57
  58
  59 def versiontuple(v):
  60     """utility for compare dot separate versions. Fills with zeros to proper number comparison"""
  61     filled = []
  62     for point in v.split("."):
  63         filled.append(point.zfill(8))
  64     return tuple(filled)
  65
  66
  67 class BaseTopic:
  68     # static variables for all instance classes
  69     topic = None        # to_override
  70     topic_msg = None    # to_override
  71     schema_new = None   # to_override
  72     schema_edit = None  # to_override
  73     multiproject = True  # True if this Topic can be shared by several projects. Then it contains _admin.projects_read
  74
  75     default_quota = 500
  76
  77     # Alternative ID Fields for some Topics
  78     alt_id_field = {
  79         "projects": "name",
  80         "users": "username",
  81         "roles": "name"
  82     }
  83
  84     def __init__(self, db, fs, msg, auth):
  85         self.db = db
  86         self.fs = fs
  87         self.msg = msg
  88         self.logger = logging.getLogger("nbi.engine")
  89         self.auth = auth
  90
  91     @staticmethod
  92     def id_field(topic, value):
  93         """Returns ID Field for given topic and field value"""
  94         if topic in BaseTopic.alt_id_field.keys() and not is_valid_uuid(value):
  95             return BaseTopic.alt_id_field[topic]
  96         else:
  97             return "_id"
  98
  99     @staticmethod
 100     def _remove_envelop(indata=None):
 101         if not indata:
 102             return {}
 103         return indata
 104
 105     def check_quota(self, session):
 106         """
 107         Check whether topic quota is exceeded by the given project
 108         Used by relevant topics' 'new' function to decide whether or not creation of the new item should be allowed
 109         :param projects: projects (tuple) for which quota should be checked
 110         :param override: boolean. If true, don't raise ValidationError even though quota be exceeded
 111         :return: None
 112         :raise:
 113             DbException if project not found
 114             ValidationError if quota exceeded and not overridden
 115         """
 116         if session["force"] or session["admin"]:
 117             return
 118         projects = session["project_id"]
 119         for project in projects:
 120             proj = self.auth.get_project(project)
 121             pid = proj["_id"]
 122             quota = proj.get("quotas", {}).get(self.topic, self.default_quota)
 123             count = self.db.count(self.topic, {"_admin.projects_read": pid})
 124             if count >= quota:
 125                 name = proj["name"]
 126                 raise ValidationError("{} quota ({}) exceeded for project {} ({})".format(self.topic, quota, name, pid))
 127
 128     def _validate_input_new(self, input, force=False):
 129         """
 130         Validates input user content for a new entry. It uses jsonschema. Some overrides will use pyangbind
 131         :param input: user input content for the new topic
 132         :param force: may be used for being more tolerant
 133         :return: The same input content, or a changed version of it.
 134         """
 135         if self.schema_new:
 136             validate_input(input, self.schema_new)
 137         return input
 138
 139     def _validate_input_edit(self, input, force=False):
 140         """
 141         Validates input user content for an edition. It uses jsonschema. Some overrides will use pyangbind
 142         :param input: user input content for the new topic
 143         :param force: may be used for being more tolerant
 144         :return: The same input content, or a changed version of it.
 145         """
 146         if self.schema_edit:
 147             validate_input(input, self.schema_edit)
 148         return input
 149
 150     @staticmethod
 151     def _get_project_filter(session):
 152         """
 153         Generates a filter dictionary for querying database, so that only allowed items for this project can be
 154         addressed. Only propietary or public can be used. Allowed projects are at _admin.project_read/write. If it is
 155         not present or contains ANY mean public.
 156         :param session: contains:
 157             project_id: project list this session has rights to access. Can be empty, one or several
 158             set_project: items created will contain this project list
 159             force: True or False
 160             public: True, False or None
 161             method: "list", "show", "write", "delete"
 162             admin: True or False
 163         :return: dictionary with project filter
 164         """
 165         p_filter = {}
 166         project_filter_n = []
 167         project_filter = list(session["project_id"])
 168
 169         if session["method"] not in ("list", "delete"):
 170             if project_filter:
 171                 project_filter.append("ANY")
 172         elif session["public"] is not None:
 173             if session["public"]:
 174                 project_filter.append("ANY")
 175             else:
 176                 project_filter_n.append("ANY")
 177
 178         if session.get("PROJECT.ne"):
 179             project_filter_n.append(session["PROJECT.ne"])
 180
 181         if project_filter:
 182             if session["method"] in ("list", "show", "delete") or session.get("set_project"):
 183                 p_filter["_admin.projects_read.cont"] = project_filter
 184             else:
 185                 p_filter["_admin.projects_write.cont"] = project_filter
 186         if project_filter_n:
 187             if session["method"] in ("list", "show", "delete") or session.get("set_project"):
 188                 p_filter["_admin.projects_read.ncont"] = project_filter_n
 189             else:
 190                 p_filter["_admin.projects_write.ncont"] = project_filter_n
 191
 192         return p_filter
 193
 194     def check_conflict_on_new(self, session, indata):
 195         """
 196         Check that the data to be inserted is valid
 197         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 198         :param indata: data to be inserted
 199         :return: None or raises EngineException
 200         """
 201         pass
 202
 203     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 204         """
 205         Check that the data to be edited/uploaded is valid
 206         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 207         :param final_content: data once modified. This method may change it.
 208         :param edit_content: incremental data that contains the modifications to apply
 209         :param _id: internal _id
 210         :return: None or raises EngineException
 211         """
 212         if not self.multiproject:
 213             return
 214         # Change public status
 215         if session["public"] is not None:
 216             if session["public"] and "ANY" not in final_content["_admin"]["projects_read"]:
 217                 final_content["_admin"]["projects_read"].append("ANY")
 218                 final_content["_admin"]["projects_write"].clear()
 219             if not session["public"] and "ANY" in final_content["_admin"]["projects_read"]:
 220                 final_content["_admin"]["projects_read"].remove("ANY")
 221
 222         # Change project status
 223         if session.get("set_project"):
 224             for p in session["set_project"]:
 225                 if p not in final_content["_admin"]["projects_read"]:
 226                     final_content["_admin"]["projects_read"].append(p)
 227
 228     def check_unique_name(self, session, name, _id=None):
 229         """
 230         Check that the name is unique for this project
 231         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 232         :param name: name to be checked
 233         :param _id: If not None, ignore this entry that are going to change
 234         :return: None or raises EngineException
 235         """
 236         if not self.multiproject:
 237             _filter = {}
 238         else:
 239             _filter = self._get_project_filter(session)
 240         _filter["name"] = name
 241         if _id:
 242             _filter["_id.neq"] = _id
 243         if self.db.get_one(self.topic, _filter, fail_on_empty=False, fail_on_more=False):
 244             raise EngineException("name '{}' already exists for {}".format(name, self.topic), HTTPStatus.CONFLICT)
 245
 246     @staticmethod
 247     def format_on_new(content, project_id=None, make_public=False):
 248         """
 249         Modifies content descriptor to include _admin
 250         :param content: descriptor to be modified
 251         :param project_id: if included, it add project read/write permissions. Can be None or a list
 252         :param make_public: if included it is generated as public for reading.
 253         :return: op_id: operation id on asynchronous operation, None otherwise. In addition content is modified
 254         """
 255         now = time()
 256         if "_admin" not in content:
 257             content["_admin"] = {}
 258         if not content["_admin"].get("created"):
 259             content["_admin"]["created"] = now
 260         content["_admin"]["modified"] = now
 261         if not content.get("_id"):
 262             content["_id"] = str(uuid4())
 263         if project_id is not None:
 264             if not content["_admin"].get("projects_read"):
 265                 content["_admin"]["projects_read"] = list(project_id)
 266                 if make_public:
 267                     content["_admin"]["projects_read"].append("ANY")
 268             if not content["_admin"].get("projects_write"):
 269                 content["_admin"]["projects_write"] = list(project_id)
 270         return None
 271
 272     @staticmethod
 273     def format_on_edit(final_content, edit_content):
 274         """
 275         Modifies final_content to admin information upon edition
 276         :param final_content: final content to be stored at database
 277         :param edit_content: user requested update content
 278         :return: operation id, if this edit implies an asynchronous operation; None otherwise
 279         """
 280         if final_content.get("_admin"):
 281             now = time()
 282             final_content["_admin"]["modified"] = now
 283         return None
 284
 285     def _send_msg(self, action, content):
 286         if self.topic_msg:
 287             content.pop("_admin", None)
 288             self.msg.write(self.topic_msg, action, content)
 289
 290     def check_conflict_on_del(self, session, _id, db_content):
 291         """
 292         Check if deletion can be done because of dependencies if it is not force. To override
 293         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 294         :param _id: internal _id
 295         :param db_content: The database content of this item _id
 296         :return: None if ok or raises EngineException with the conflict
 297         """
 298         pass
 299
 300     @staticmethod
 301     def _update_input_with_kwargs(desc, kwargs):
 302         """
 303         Update descriptor with the kwargs. It contains dot separated keys
 304         :param desc: dictionary to be updated
 305         :param kwargs: plain dictionary to be used for updating.
 306         :return: None, 'desc' is modified. It raises EngineException.
 307         """
 308         if not kwargs:
 309             return
 310         try:
 311             for k, v in kwargs.items():
 312                 update_content = desc
 313                 kitem_old = None
 314                 klist = k.split(".")
 315                 for kitem in klist:
 316                     if kitem_old is not None:
 317                         update_content = update_content[kitem_old]
 318                     if isinstance(update_content, dict):
 319                         kitem_old = kitem
 320                     elif isinstance(update_content, list):
 321                         kitem_old = int(kitem)
 322                     else:
 323                         raise EngineException(
 324                             "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 325                 update_content[kitem_old] = v
 326         except KeyError:
 327             raise EngineException(
 328                 "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 329         except ValueError:
 330             raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 331                 k, kitem))
 332         except IndexError:
 333             raise EngineException(
 334                 "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 335
 336     def show(self, session, _id):
 337         """
 338         Get complete information on an topic
 339         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 340         :param _id: server internal id
 341         :return: dictionary, raise exception if not found.
 342         """
 343         if not self.multiproject:
 344             filter_db = {}
 345         else:
 346             filter_db = self._get_project_filter(session)
 347         # To allow project&user addressing by name AS WELL AS _id
 348         filter_db[BaseTopic.id_field(self.topic, _id)] = _id
 349         return self.db.get_one(self.topic, filter_db)
 350         # TODO transform data for SOL005 URL requests
 351         # TODO remove _admin if not admin
 352
 353     def get_file(self, session, _id, path=None, accept_header=None):
 354         """
 355         Only implemented for descriptor topics. Return the file content of a descriptor
 356         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 357         :param _id: Identity of the item to get content
 358         :param path: artifact path or "$DESCRIPTOR" or None
 359         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 360         :return: opened file or raises an exception
 361         """
 362         raise EngineException("Method get_file not valid for this topic", HTTPStatus.INTERNAL_SERVER_ERROR)
 363
 364     def list(self, session, filter_q=None):
 365         """
 366         Get a list of the topic that matches a filter
 367         :param session: contains the used login username and working project
 368         :param filter_q: filter of data to be applied
 369         :return: The list, it can be empty if no one match the filter.
 370         """
 371         if not filter_q:
 372             filter_q = {}
 373         if self.multiproject:
 374             filter_q.update(self._get_project_filter(session))
 375
 376         # TODO transform data for SOL005 URL requests. Transform filtering
 377         # TODO implement "field-type" query string SOL005
 378         return self.db.get_list(self.topic, filter_q)
 379
 380     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 381         """
 382         Creates a new entry into database.
 383         :param rollback: list to append created items at database in case a rollback may to be done
 384         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 385         :param indata: data to be inserted
 386         :param kwargs: used to override the indata descriptor
 387         :param headers: http request headers
 388         :return: _id, op_id:
 389             _id: identity of the inserted data.
 390              op_id: operation id if this is asynchronous, None otherwise
 391         """
 392         try:
 393             if self.multiproject:
 394                 self.check_quota(session)
 395
 396             content = self._remove_envelop(indata)
 397
 398             # Override descriptor with query string kwargs
 399             self._update_input_with_kwargs(content, kwargs)
 400             content = self._validate_input_new(content, force=session["force"])
 401             self.check_conflict_on_new(session, content)
 402             op_id = self.format_on_new(content, project_id=session["project_id"], make_public=session["public"])
 403             _id = self.db.create(self.topic, content)
 404             rollback.append({"topic": self.topic, "_id": _id})
 405             if op_id:
 406                 content["op_id"] = op_id
 407             self._send_msg("created", content)
 408             return _id, op_id
 409         except ValidationError as e:
 410             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 411
 412     def upload_content(self, session, _id, indata, kwargs, headers):
 413         """
 414         Only implemented for descriptor topics.  Used for receiving content by chunks (with a transaction_id header
 415         and/or gzip file. It will store and extract)
 416         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 417         :param _id : the database id of entry to be updated
 418         :param indata: http body request
 419         :param kwargs: user query string to override parameters. NOT USED
 420         :param headers:  http request headers
 421         :return: True package has is completely uploaded or False if partial content has been uplodaed.
 422             Raise exception on error
 423         """
 424         raise EngineException("Method upload_content not valid for this topic", HTTPStatus.INTERNAL_SERVER_ERROR)
 425
 426     def delete_list(self, session, filter_q=None):
 427         """
 428         Delete a several entries of a topic. This is for internal usage and test only, not exposed to NBI API
 429         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 430         :param filter_q: filter of data to be applied
 431         :return: The deleted list, it can be empty if no one match the filter.
 432         """
 433         # TODO add admin to filter, validate rights
 434         if not filter_q:
 435             filter_q = {}
 436         if self.multiproject:
 437             filter_q.update(self._get_project_filter(session))
 438         return self.db.del_list(self.topic, filter_q)
 439
 440     def delete_extra(self, session, _id, db_content):
 441         """
 442         Delete other things apart from database entry of a item _id.
 443         e.g.: other associated elements at database and other file system storage
 444         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 445         :param _id: server internal id
 446         :param db_content: The database content of the _id. It is already deleted when reached this method, but the
 447             content is needed in same cases
 448         :return: None if ok or raises EngineException with the problem
 449         """
 450         pass
 451
 452     def delete(self, session, _id, dry_run=False):
 453         """
 454         Delete item by its internal _id
 455         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 456         :param _id: server internal id
 457         :param dry_run: make checking but do not delete
 458         :return: operation id (None if there is not operation), raise exception if error or not found, conflict, ...
 459         """
 460
 461         # To allow addressing projects and users by name AS WELL AS by _id
 462         filter_q = {BaseTopic.id_field(self.topic, _id): _id}
 463         item_content = self.db.get_one(self.topic, filter_q)
 464
 465         # TODO add admin to filter, validate rights
 466         # data = self.get_item(topic, _id)
 467         self.check_conflict_on_del(session, _id, item_content)
 468         if dry_run:
 469             return None
 470
 471         if self.multiproject:
 472             filter_q.update(self._get_project_filter(session))
 473         if self.multiproject and session["project_id"]:
 474             # remove reference from project_read. If not last delete
 475             # if this topic is not part of session["project_id"] no midification at database is done and an exception
 476             # is raised
 477             self.db.set_one(self.topic, filter_q, update_dict=None,
 478                             pull={"_admin.projects_read": {"$in": session["project_id"]}})
 479             # try to delete if there is not any more reference from projects. Ignore if it is not deleted
 480             filter_q = {'_id': _id, '_admin.projects_read': [[], ["ANY"]]}
 481             v = self.db.del_one(self.topic, filter_q, fail_on_empty=False)
 482             if not v or not v["deleted"]:
 483                 return None
 484         else:
 485             self.db.del_one(self.topic, filter_q)
 486         self.delete_extra(session, _id, item_content)
 487         self._send_msg("deleted", {"_id": _id})
 488         return None
 489
 490     def edit(self, session, _id, indata=None, kwargs=None, content=None):
 491         """
 492         Change the content of an item
 493         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 494         :param _id: server internal id
 495         :param indata: contains the changes to apply
 496         :param kwargs: modifies indata
 497         :param content: original content of the item
 498         :return: op_id: operation id if this is processed asynchronously, None otherwise
 499         """
 500         indata = self._remove_envelop(indata)
 501
 502         # Override descriptor with query string kwargs
 503         if kwargs:
 504             self._update_input_with_kwargs(indata, kwargs)
 505         try:
 506             if indata and session.get("set_project"):
 507                 raise EngineException("Cannot edit content and set to project (query string SET_PROJECT) at same time",
 508                                       HTTPStatus.UNPROCESSABLE_ENTITY)
 509             indata = self._validate_input_edit(indata, force=session["force"])
 510
 511             # TODO self._check_edition(session, indata, _id, force)
 512             if not content:
 513                 content = self.show(session, _id)
 514             deep_update_rfc7396(content, indata)
 515
 516             # To allow project addressing by name AS WELL AS _id. Get the _id, just in case the provided one is a name
 517             _id = content.get("_id") or _id
 518
 519             self.check_conflict_on_edit(session, content, indata, _id=_id)
 520             op_id = self.format_on_edit(content, indata)
 521
 522             self.db.replace(self.topic, _id, content)
 523
 524             indata.pop("_admin", None)
 525             if op_id:
 526                 indata["op_id"] = op_id
 527             indata["_id"] = _id
 528             self._send_msg("edited", indata)
 529             return op_id
 530         except ValidationError as e:
 531             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)