X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Fdbmemory.py;h=d089575ffa9b5666614b9aabe7ba1a1150555411;hb=2644b76248a1b96f7a47013b414e31b4e3feecf8;hp=e37d97b5aed3eb22efce834657a54f277905f3b3;hpb=87858cab98b3b169fc891fd2e0a0ba10f8b46127;p=osm%2Fcommon.git diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index e37d97b..d089575 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -17,6 +17,7 @@ import logging from osm_common.dbbase import DbException, DbBase +from osm_common.dbmongo import deep_update from http import HTTPStatus from uuid import uuid4 from copy import deepcopy @@ -25,9 +26,8 @@ __author__ = "Alfonso Tierno " class DbMemory(DbBase): - - def __init__(self, logger_name='db', master_password=None): - super().__init__(logger_name, master_password) + def __init__(self, logger_name="db", lock=False): + super().__init__(logger_name, lock) self.db = {} def db_connect(self, config): @@ -38,19 +38,160 @@ class DbMemory(DbBase): """ if "logger_name" in config: self.logger = logging.getLogger(config["logger_name"]) + master_key = config.get("commonkey") or config.get("masterpassword") + if master_key: + self.set_secret_key(master_key) @staticmethod def _format_filter(q_filter): - return q_filter # TODO + db_filter = {} + # split keys with ANYINDEX in this way: + # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}} + if q_filter: + for k, v in q_filter.items(): + db_v = v + kleft, _, kright = k.rpartition(".ANYINDEX.") + while kleft: + k = kleft + ".ANYINDEX" + db_v = {kright: db_v} + kleft, _, kright = k.rpartition(".ANYINDEX.") + deep_update(db_filter, {k: db_v}) + + return db_filter def _find(self, table, q_filter): + def recursive_find(key_list, key_next_index, content, oper, target): + if key_next_index == len(key_list) or content is None: + try: + if oper in ("eq", "cont"): + if isinstance(target, list): + if isinstance(content, list): + return any( + content_item in target for content_item in content + ) + return content in target + elif isinstance(content, list): + return target in content + else: + return content == target + elif oper in ("neq", "ne", "ncont"): + if isinstance(target, list): + if isinstance(content, list): + return all( + content_item not in target + for content_item in content + ) + return content not in target + elif isinstance(content, list): + return target not in content + else: + return content != target + if oper == "gt": + return content > target + elif oper == "gte": + return content >= target + elif oper == "lt": + return content < target + elif oper == "lte": + return content <= target + else: + raise DbException( + "Unknown filter operator '{}' in key '{}'".format( + oper, ".".join(key_list) + ), + http_code=HTTPStatus.BAD_REQUEST, + ) + except TypeError: + return False + + elif isinstance(content, dict): + return recursive_find( + key_list, + key_next_index + 1, + content.get(key_list[key_next_index]), + oper, + target, + ) + elif isinstance(content, list): + look_for_match = True # when there is a match return immediately + if (target is None) != ( + oper in ("neq", "ne", "ncont") + ): # one True and other False (Xor) + look_for_match = ( + False # when there is not a match return immediately + ) + + for content_item in content: + if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict): + matches = True + for k2, v2 in target.items(): + k_new_list = k2.split(".") + new_operator = "eq" + if k_new_list[-1] in ( + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "cont", + "ncont", + "neq", + ): + new_operator = k_new_list.pop() + if not recursive_find( + k_new_list, 0, content_item, new_operator, v2 + ): + matches = False + break + + else: + matches = recursive_find( + key_list, key_next_index, content_item, oper, target + ) + if matches == look_for_match: + return matches + if key_list[key_next_index].isdecimal() and int( + key_list[key_next_index] + ) < len(content): + matches = recursive_find( + key_list, + key_next_index + 1, + content[int(key_list[key_next_index])], + oper, + target, + ) + if matches == look_for_match: + return matches + return not look_for_match + else: # content is not dict, nor list neither None, so not found + if oper in ("neq", "ne", "ncont"): + return target is not None + else: + return target is None + for i, row in enumerate(self.db.get(table, ())): - match = True - if q_filter: - for k, v in q_filter.items(): - if k not in row or v != row[k]: - match = False - if match: + q_filter = q_filter or {} + for k, v in q_filter.items(): + k_list = k.split(".") + operator = "eq" + if k_list[-1] in ( + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "cont", + "ncont", + "neq", + ): + operator = k_list.pop() + matches = recursive_find(k_list, 0, row, operator, v) + if not matches: + break + else: + # match yield i, row def get_list(self, table, q_filter=None): @@ -62,14 +203,31 @@ class DbMemory(DbBase): """ try: result = [] - for _, row in self._find(table, self._format_filter(q_filter)): - result.append(deepcopy(row)) + with self.lock: + for _, row in self._find(table, self._format_filter(q_filter)): + result.append(deepcopy(row)) return result except DbException: raise except Exception as e: # TODO refine raise DbException(str(e)) + def count(self, table, q_filter=None): + """ + Count the number of entries matching q_filter + :param table: collection or table + :param q_filter: Filter + :return: number of entries found (can be zero) + :raise: DbException on error + """ + try: + with self.lock: + return sum(1 for x in self._find(table, self._format_filter(q_filter))) + except DbException: + raise + except Exception as e: # TODO refine + raise DbException(str(e)) + def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True): """ Obtain one entry matching q_filter @@ -83,15 +241,23 @@ class DbMemory(DbBase): """ try: result = None - for _, row in self._find(table, self._format_filter(q_filter)): - if not fail_on_more: - return deepcopy(row) - if result: - raise DbException("Found more than one entry with filter='{}'".format(q_filter), - HTTPStatus.CONFLICT.value) - result = row + with self.lock: + for _, row in self._find(table, self._format_filter(q_filter)): + if not fail_on_more: + return deepcopy(row) + if result: + raise DbException( + "Found more than one entry with filter='{}'".format( + q_filter + ), + HTTPStatus.CONFLICT.value, + ) + result = row if not result and fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + raise DbException( + "Not found entry with filter='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) return deepcopy(result) except Exception as e: # TODO refine raise DbException(str(e)) @@ -105,8 +271,9 @@ class DbMemory(DbBase): """ try: id_list = [] - for i, _ in self._find(table, self._format_filter(q_filter)): - id_list.append(i) + with self.lock: + for i, _ in self._find(table, self._format_filter(q_filter)): + id_list.append(i) deleted = len(id_list) for i in reversed(id_list): del self.db[table][i] @@ -126,17 +293,291 @@ class DbMemory(DbBase): :return: Dict with the number of entries deleted """ try: - for i, _ in self._find(table, self._format_filter(q_filter)): - break - else: - if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) - return None - del self.db[table][i] + with self.lock: + for i, _ in self._find(table, self._format_filter(q_filter)): + break + else: + if fail_on_empty: + raise DbException( + "Not found entry with filter='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) + return None + del self.db[table][i] return {"deleted": 1} except Exception as e: # TODO refine raise DbException(str(e)) + def _update( + self, + db_item, + update_dict, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): + """ + Modifies an entry at database + :param db_item: entry of the table to update + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is + ignored. If not exist, it is ignored + :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value + if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array + :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value + is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array + :return: True if database has been changed, False if not; Exception on error + """ + + def _iterate_keys(k, db_nested, populate=True): + k_list = k.split(".") + k_item_prev = k_list[0] + populated = False + if k_item_prev not in db_nested and populate: + populated = True + db_nested[k_item_prev] = None + for k_item in k_list[1:]: + if isinstance(db_nested[k_item_prev], dict): + if k_item not in db_nested[k_item_prev]: + if not populate: + raise DbException( + "Cannot set '{}', not existing '{}'".format(k, k_item) + ) + populated = True + db_nested[k_item_prev][k_item] = None + elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): + # extend list with Nones if index greater than list + k_item = int(k_item) + if k_item >= len(db_nested[k_item_prev]): + if not populate: + raise DbException( + "Cannot set '{}', index too large '{}'".format( + k, k_item + ) + ) + populated = True + db_nested[k_item_prev] += [None] * ( + k_item - len(db_nested[k_item_prev]) + 1 + ) + elif db_nested[k_item_prev] is None: + if not populate: + raise DbException( + "Cannot set '{}', not existing '{}'".format(k, k_item) + ) + populated = True + db_nested[k_item_prev] = {k_item: None} + else: # number, string, boolean, ... or list but with not integer key + raise DbException( + "Cannot set '{}' on existing '{}={}'".format( + k, k_item_prev, db_nested[k_item_prev] + ) + ) + db_nested = db_nested[k_item_prev] + k_item_prev = k_item + return db_nested, k_item_prev, populated + + updated = False + try: + if update_dict: + for dot_k, v in update_dict.items(): + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item) + dict_to_update[key_to_update] = v + updated = True + if unset: + for dot_k in unset: + try: + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) + del dict_to_update[key_to_update] + updated = True + except Exception: + pass + if pull: + for dot_k, v in pull.items(): + try: + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot pull '{}'. Target is not a list".format(dot_k) + ) + while v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(v) + updated = True + if pull_list: + for dot_k, v in pull_list.items(): + if not isinstance(v, list): + raise DbException( + "Invalid content at pull_list, '{}' must be an array".format( + dot_k + ), + http_code=HTTPStatus.BAD_REQUEST, + ) + try: + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot pull_list '{}'. Target is not a list".format(dot_k) + ) + for single_v in v: + while single_v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(single_v) + updated = True + if push: + for dot_k, v in push.items(): + dict_to_update, key_to_update, populated = _iterate_keys( + dot_k, db_item + ) + if ( + isinstance(dict_to_update, dict) + and key_to_update not in dict_to_update + ): + dict_to_update[key_to_update] = [v] + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = [v] + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot push '{}'. Target is not a list".format(dot_k) + ) + else: + dict_to_update[key_to_update].append(v) + updated = True + if push_list: + for dot_k, v in push_list.items(): + if not isinstance(v, list): + raise DbException( + "Invalid content at push_list, '{}' must be an array".format( + dot_k + ), + http_code=HTTPStatus.BAD_REQUEST, + ) + dict_to_update, key_to_update, populated = _iterate_keys( + dot_k, db_item + ) + if ( + isinstance(dict_to_update, dict) + and key_to_update not in dict_to_update + ): + dict_to_update[key_to_update] = v.copy() + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = v.copy() + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot push '{}'. Target is not a list".format(dot_k), + http_code=HTTPStatus.CONFLICT, + ) + else: + dict_to_update[key_to_update] += v + updated = True + + return updated + except DbException: + raise + except Exception as e: # TODO refine + raise DbException(str(e)) + + def set_one( + self, + table, + q_filter, + update_dict, + fail_on_empty=True, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): + """ + Modifies an entry at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is + ignored. If not exist, it is ignored + :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value + if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array + :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value + is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array + :return: Dict with the number of entries modified. None if no matching is found. + """ + with self.lock: + for i, db_item in self._find(table, self._format_filter(q_filter)): + updated = self._update( + db_item, + update_dict, + unset=unset, + pull=pull, + push=push, + push_list=push_list, + pull_list=pull_list, + ) + return {"updated": 1 if updated else 0} + else: + if fail_on_empty: + raise DbException( + "Not found entry with _id='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) + return None + + def set_list( + self, + table, + q_filter, + update_dict, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): + """Modifies al matching entries at database. Same as push. Do not fail if nothing matches""" + with self.lock: + updated = 0 + found = 0 + for _, db_item in self._find(table, self._format_filter(q_filter)): + found += 1 + if self._update( + db_item, + update_dict, + unset=unset, + pull=pull, + push=push, + push_list=push_list, + pull_list=pull_list, + ): + updated += 1 + # if not found and fail_on_empty: + # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return {"updated": updated} if found else None + def replace(self, table, _id, indata, fail_on_empty=True): """ Replace the content of an entry @@ -148,14 +589,20 @@ class DbMemory(DbBase): :return: Dict with the number of entries replaced """ try: - for i, _ in self._find(table, self._format_filter({"_id": _id})): - break - else: - if fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(_id), HTTPStatus.NOT_FOUND) - return None - self.db[table][i] = deepcopy(indata) + with self.lock: + for i, _ in self._find(table, self._format_filter({"_id": _id})): + break + else: + if fail_on_empty: + raise DbException( + "Not found entry with _id='{}'".format(_id), + HTTPStatus.NOT_FOUND, + ) + return None + self.db[table][i] = deepcopy(indata) return {"updated": 1} + except DbException: + raise except Exception as e: # TODO refine raise DbException(str(e)) @@ -164,22 +611,47 @@ class DbMemory(DbBase): Add a new entry at database :param table: collection or table :param indata: content to be added - :return: database id of the inserted element. Raises a DbException on error + :return: database '_id' of the inserted element. Raises a DbException on error """ try: id = indata.get("_id") if not id: id = str(uuid4()) indata["_id"] = id - if table not in self.db: - self.db[table] = [] - self.db[table].append(deepcopy(indata)) + with self.lock: + if table not in self.db: + self.db[table] = [] + self.db[table].append(deepcopy(indata)) return id except Exception as e: # TODO refine raise DbException(str(e)) + def create_list(self, table, indata_list): + """ + Add a new entry at database + :param table: collection or table + :param indata_list: list content to be added + :return: list of inserted 'id's. Raises a DbException on error + """ + try: + _ids = [] + with self.lock: + for indata in indata_list: + _id = indata.get("_id") + if not _id: + _id = str(uuid4()) + indata["_id"] = _id + with self.lock: + if table not in self.db: + self.db[table] = [] + self.db[table].append(deepcopy(indata)) + _ids.append(_id) + return _ids + except Exception as e: # TODO refine + raise DbException(str(e)) + -if __name__ == '__main__': +if __name__ == "__main__": # some test code db = DbMemory() db.create("test", {"_id": 1, "data": 1})