osm_nbi/base_topic.py

   1 # -*- coding: utf-8 -*-
   2
   3 import logging
   4 from uuid import uuid4
   5 from http import HTTPStatus
   6 from time import time
   7 from osm_common.dbbase import deep_update_rfc7396
   8 from validation import validate_input, ValidationError
   9
  10 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  11
  12
  13 class EngineException(Exception):
  14
  15     def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
  16         self.http_code = http_code
  17         Exception.__init__(self, message)
  18
  19
  20 def get_iterable(input_var):
  21     """
  22     Returns an iterable, in case input_var is None it just returns an empty tuple
  23     :param input_var: can be a list, tuple or None
  24     :return: input_var or () if it is None
  25     """
  26     if input_var is None:
  27         return ()
  28     return input_var
  29
  30
  31 def versiontuple(v):
  32     """utility for compare dot separate versions. Fills with zeros to proper number comparison"""
  33     filled = []
  34     for point in v.split("."):
  35         filled.append(point.zfill(8))
  36     return tuple(filled)
  37
  38
  39 class BaseTopic:
  40     # static variables for all instance classes
  41     topic = None        # to_override
  42     topic_msg = None    # to_override
  43     schema_new = None   # to_override
  44     schema_edit = None  # to_override
  45
  46     def __init__(self, db, fs, msg):
  47         self.db = db
  48         self.fs = fs
  49         self.msg = msg
  50         self.logger = logging.getLogger("nbi.engine")
  51
  52     @staticmethod
  53     def _remove_envelop(indata=None):
  54         if not indata:
  55             return {}
  56         return indata
  57
  58     def _validate_input_new(self, input, force=False):
  59         """
  60         Validates input user content for a new entry. It uses jsonschema. Some overrides will use pyangbind
  61         :param input: user input content for the new topic
  62         :param force: may be used for being more tolerant
  63         :return: The same input content, or a changed version of it.
  64         """
  65         if self.schema_new:
  66             validate_input(input, self.schema_new)
  67         return input
  68
  69     def _validate_input_edit(self, input, force=False):
  70         """
  71         Validates input user content for an edition. It uses jsonschema. Some overrides will use pyangbind
  72         :param input: user input content for the new topic
  73         :param force: may be used for being more tolerant
  74         :return: The same input content, or a changed version of it.
  75         """
  76         if self.schema_edit:
  77             validate_input(input, self.schema_edit)
  78         return input
  79
  80     @staticmethod
  81     def _get_project_filter(session, write=False, show_all=True):
  82         """
  83         Generates a filter dictionary for querying database, so that only allowed items for this project can be
  84         addressed. Only propietary or public can be used. Allowed projects are at _admin.project_read/write. If it is
  85         not present or contains ANY mean public.
  86         :param session: contains "username", if user is "admin" and the working "project_id"
  87         :param write: if operation is for reading (False) or writing (True)
  88         :param show_all:  if True it will show public or
  89         :return:
  90         """
  91         if write:
  92             k = "_admin.projects_write.cont"
  93         else:
  94             k = "_admin.projects_read.cont"
  95         if not show_all:
  96             return {k: session["project_id"]}
  97         elif session["admin"]:   # and show_all:  # allows all
  98             return {}
  99         else:
 100             return {k: ["ANY", session["project_id"], None]}
 101
 102     def check_conflict_on_new(self, session, indata, force=False):
 103         """
 104         Check that the data to be inserted is valid
 105         :param session: contains "username", if user is "admin" and the working "project_id"
 106         :param indata: data to be inserted
 107         :param force: boolean. With force it is more tolerant
 108         :return: None or raises EngineException
 109         """
 110         pass
 111
 112     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 113         """
 114         Check that the data to be edited/uploaded is valid
 115         :param session: contains "username", if user is "admin" and the working "project_id"
 116         :param final_content: data once modified
 117         :param edit_content: incremental data that contains the modifications to apply
 118         :param _id: internal _id
 119         :param force: boolean. With force it is more tolerant
 120         :return: None or raises EngineException
 121         """
 122         pass
 123
 124     def check_unique_name(self, session, name, _id=None):
 125         """
 126         Check that the name is unique for this project
 127         :param session: contains "username", if user is "admin" and the working "project_id"
 128         :param name: name to be checked
 129         :param _id: If not None, ignore this entry that are going to change
 130         :return: None or raises EngineException
 131         """
 132         _filter = self._get_project_filter(session, write=False, show_all=False)
 133         _filter["name"] = name
 134         if _id:
 135             _filter["_id.neq"] = _id
 136         if self.db.get_one(self.topic, _filter, fail_on_empty=False, fail_on_more=False):
 137             raise EngineException("name '{}' already exists for {}".format(name, self.topic), HTTPStatus.CONFLICT)
 138
 139     @staticmethod
 140     def format_on_new(content, project_id=None, make_public=False):
 141         """
 142         Modifies content descriptor to include _admin
 143         :param content: descriptor to be modified
 144         :param project_id: if included, it add project read/write permissions
 145         :param make_public: if included it is generated as public for reading.
 146         :return: None, but content is modified
 147         """
 148         now = time()
 149         if "_admin" not in content:
 150             content["_admin"] = {}
 151         if not content["_admin"].get("created"):
 152             content["_admin"]["created"] = now
 153         content["_admin"]["modified"] = now
 154         if not content.get("_id"):
 155             content["_id"] = str(uuid4())
 156         if project_id:
 157             if not content["_admin"].get("projects_read"):
 158                 content["_admin"]["projects_read"] = [project_id]
 159                 if make_public:
 160                     content["_admin"]["projects_read"].append("ANY")
 161             if not content["_admin"].get("projects_write"):
 162                 content["_admin"]["projects_write"] = [project_id]
 163
 164     @staticmethod
 165     def format_on_edit(final_content, edit_content):
 166         if final_content.get("_admin"):
 167             now = time()
 168             final_content["_admin"]["modified"] = now
 169
 170     def _send_msg(self, action, content):
 171         if self.topic_msg:
 172             content.pop("_admin", None)
 173             self.msg.write(self.topic_msg, action, content)
 174
 175     def check_conflict_on_del(self, session, _id, force=False):
 176         """
 177         Check if deletion can be done because of dependencies if it is not force. To override
 178         :param session: contains "username", if user is "admin" and the working "project_id"
 179         :param _id: itnernal _id
 180         :param force: Avoid this checking
 181         :return: None if ok or raises EngineException with the conflict
 182         """
 183         pass
 184
 185     @staticmethod
 186     def _update_input_with_kwargs(desc, kwargs):
 187         """
 188         Update descriptor with the kwargs. It contains dot separated keys
 189         :param desc: dictionary to be updated
 190         :param kwargs: plain dictionary to be used for updating.
 191         :return: None, 'desc' is modified. It raises EngineException.
 192         """
 193         if not kwargs:
 194             return
 195         try:
 196             for k, v in kwargs.items():
 197                 update_content = desc
 198                 kitem_old = None
 199                 klist = k.split(".")
 200                 for kitem in klist:
 201                     if kitem_old is not None:
 202                         update_content = update_content[kitem_old]
 203                     if isinstance(update_content, dict):
 204                         kitem_old = kitem
 205                     elif isinstance(update_content, list):
 206                         kitem_old = int(kitem)
 207                     else:
 208                         raise EngineException(
 209                             "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k, kitem))
 210                 update_content[kitem_old] = v
 211         except KeyError:
 212             raise EngineException(
 213                 "Invalid query string '{}'. Descriptor does not contain '{}'".format(k, kitem_old))
 214         except ValueError:
 215             raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
 216                 k, kitem))
 217         except IndexError:
 218             raise EngineException(
 219                 "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 220
 221     def show(self, session, _id):
 222         """
 223         Get complete information on an topic
 224         :param session: contains the used login username and working project
 225         :param _id: server internal id
 226         :return: dictionary, raise exception if not found.
 227         """
 228         filter_db = self._get_project_filter(session, write=False, show_all=True)
 229         filter_db["_id"] = _id
 230         return self.db.get_one(self.topic, filter_db)
 231         # TODO transform data for SOL005 URL requests
 232         # TODO remove _admin if not admin
 233
 234     def get_file(self, session, _id, path=None, accept_header=None):
 235         """
 236         Only implemented for descriptor topics. Return the file content of a descriptor
 237         :param session: contains the used login username and working project
 238         :param _id: Identity of the item to get content
 239         :param path: artifact path or "$DESCRIPTOR" or None
 240         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 241         :return: opened file or raises an exception
 242         """
 243         raise EngineException("Method get_file not valid for this topic", HTTPStatus.INTERNAL_SERVER_ERROR)
 244
 245     def list(self, session, filter_q=None):
 246         """
 247         Get a list of the topic that matches a filter
 248         :param session: contains the used login username and working project
 249         :param filter_q: filter of data to be applied
 250         :return: The list, it can be empty if no one match the filter.
 251         """
 252         if not filter_q:
 253             filter_q = {}
 254
 255         filter_q.update(self._get_project_filter(session, write=False, show_all=True))
 256
 257         # TODO transform data for SOL005 URL requests. Transform filtering
 258         # TODO implement "field-type" query string SOL005
 259         return self.db.get_list(self.topic, filter_q)
 260
 261     def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
 262         """
 263         Creates a new entry into database.
 264         :param rollback: list to append created items at database in case a rollback may to be done
 265         :param session: contains the used login username and working project
 266         :param indata: data to be inserted
 267         :param kwargs: used to override the indata descriptor
 268         :param headers: http request headers
 269         :param force: If True avoid some dependence checks
 270         :param make_public: Make the created item public to all projects
 271         :return: _id: identity of the inserted data.
 272         """
 273         try:
 274             content = self._remove_envelop(indata)
 275
 276             # Override descriptor with query string kwargs
 277             self._update_input_with_kwargs(content, kwargs)
 278             content = self._validate_input_new(content, force=force)
 279             self.check_conflict_on_new(session, content, force=force)
 280             self.format_on_new(content, project_id=session["project_id"], make_public=make_public)
 281             _id = self.db.create(self.topic, content)
 282             rollback.append({"topic": self.topic, "_id": _id})
 283             self._send_msg("create", content)
 284             return _id
 285         except ValidationError as e:
 286             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 287
 288     def upload_content(self, session, _id, indata, kwargs, headers, force=False):
 289         """
 290         Only implemented for descriptor topics.  Used for receiving content by chunks (with a transaction_id header
 291         and/or gzip file. It will store and extract)
 292         :param session: session
 293         :param _id : the database id of entry to be updated
 294         :param indata: http body request
 295         :param kwargs: user query string to override parameters. NOT USED
 296         :param headers:  http request headers
 297         :param force: to be more tolerant with validation
 298         :return: True package has is completely uploaded or False if partial content has been uplodaed.
 299             Raise exception on error
 300         """
 301         raise EngineException("Method upload_content not valid for this topic", HTTPStatus.INTERNAL_SERVER_ERROR)
 302
 303     def delete_list(self, session, filter_q=None):
 304         """
 305         Delete a several entries of a topic. This is for internal usage and test only, not exposed to NBI API
 306         :param session: contains the used login username and working project
 307         :param filter_q: filter of data to be applied
 308         :return: The deleted list, it can be empty if no one match the filter.
 309         """
 310         # TODO add admin to filter, validate rights
 311         if not filter_q:
 312             filter_q = {}
 313         filter_q.update(self._get_project_filter(session, write=True, show_all=True))
 314         return self.db.del_list(self.topic, filter_q)
 315
 316     def delete(self, session, _id, force=False, dry_run=False):
 317         """
 318         Delete item by its internal _id
 319         :param session: contains the used login username, working project, and admin rights
 320         :param _id: server internal id
 321         :param force: indicates if deletion must be forced in case of conflict
 322         :param dry_run: make checking but do not delete
 323         :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
 324         """
 325         # TODO add admin to filter, validate rights
 326         # data = self.get_item(topic, _id)
 327         self.check_conflict_on_del(session, _id, force)
 328         filter_q = self._get_project_filter(session, write=True, show_all=True)
 329         filter_q["_id"] = _id
 330         if not dry_run:
 331             v = self.db.del_one(self.topic, filter_q)
 332             self._send_msg("deleted", {"_id": _id})
 333             return v
 334         return None
 335
 336     def edit(self, session, _id, indata=None, kwargs=None, force=False, content=None):
 337         indata = self._remove_envelop(indata)
 338
 339         # Override descriptor with query string kwargs
 340         if kwargs:
 341             self._update_input_with_kwargs(indata, kwargs)
 342         try:
 343             indata = self._validate_input_edit(indata, force=force)
 344
 345             # TODO self._check_edition(session, indata, _id, force)
 346             if not content:
 347                 content = self.show(session, _id)
 348             deep_update_rfc7396(content, indata)
 349             self.check_conflict_on_edit(session, content, indata, _id=_id, force=force)
 350             self.format_on_edit(content, indata)
 351             self.db.replace(self.topic, _id, content)
 352
 353             indata.pop("_admin", None)
 354             indata["_id"] = _id
 355             self._send_msg("edit", indata)
 356             return id
 357         except ValidationError as e:
 358             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)