X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Fdbmemory.py;h=d089575ffa9b5666614b9aabe7ba1a1150555411;hb=2644b76248a1b96f7a47013b414e31b4e3feecf8;hp=c57f932f29e6e335af34c8b4171cf209777e56d3;hpb=bf6c5729597476199627c6750e8db3a92ce47c9c;p=osm%2Fcommon.git diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index c57f932..d089575 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -26,8 +26,7 @@ __author__ = "Alfonso Tierno " class DbMemory(DbBase): - - def __init__(self, logger_name='db', lock=False): + def __init__(self, logger_name="db", lock=False): super().__init__(logger_name, lock) self.db = {} @@ -61,14 +60,15 @@ class DbMemory(DbBase): return db_filter def _find(self, table, q_filter): - def recursive_find(key_list, key_next_index, content, oper, target): if key_next_index == len(key_list) or content is None: try: if oper in ("eq", "cont"): if isinstance(target, list): if isinstance(content, list): - return any(content_item in target for content_item in content) + return any( + content_item in target for content_item in content + ) return content in target elif isinstance(content, list): return target in content @@ -77,7 +77,10 @@ class DbMemory(DbBase): elif oper in ("neq", "ne", "ncont"): if isinstance(target, list): if isinstance(content, list): - return all(content_item not in target for content_item in content) + return all( + content_item not in target + for content_item in content + ) return content not in target elif isinstance(content, list): return target not in content @@ -92,18 +95,31 @@ class DbMemory(DbBase): elif oper == "lte": return content <= target else: - raise DbException("Unknown filter operator '{}' in key '{}'". - format(oper, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST) + raise DbException( + "Unknown filter operator '{}' in key '{}'".format( + oper, ".".join(key_list) + ), + http_code=HTTPStatus.BAD_REQUEST, + ) except TypeError: return False elif isinstance(content, dict): - return recursive_find(key_list, key_next_index + 1, content.get(key_list[key_next_index]), oper, - target) + return recursive_find( + key_list, + key_next_index + 1, + content.get(key_list[key_next_index]), + oper, + target, + ) elif isinstance(content, list): look_for_match = True # when there is a match return immediately - if (target is None) != (oper in ("neq", "ne", "ncont")): # one True and other False (Xor) - look_for_match = False # when there is not a match return immediately + if (target is None) != ( + oper in ("neq", "ne", "ncont") + ): # one True and other False (Xor) + look_for_match = ( + False # when there is not a match return immediately + ) for content_item in content: if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict): @@ -111,19 +127,40 @@ class DbMemory(DbBase): for k2, v2 in target.items(): k_new_list = k2.split(".") new_operator = "eq" - if k_new_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): + if k_new_list[-1] in ( + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "cont", + "ncont", + "neq", + ): new_operator = k_new_list.pop() - if not recursive_find(k_new_list, 0, content_item, new_operator, v2): + if not recursive_find( + k_new_list, 0, content_item, new_operator, v2 + ): matches = False break else: - matches = recursive_find(key_list, key_next_index, content_item, oper, target) + matches = recursive_find( + key_list, key_next_index, content_item, oper, target + ) if matches == look_for_match: return matches - if key_list[key_next_index].isdecimal() and int(key_list[key_next_index]) < len(content): - matches = recursive_find(key_list, key_next_index + 1, content[int(key_list[key_next_index])], - oper, target) + if key_list[key_next_index].isdecimal() and int( + key_list[key_next_index] + ) < len(content): + matches = recursive_find( + key_list, + key_next_index + 1, + content[int(key_list[key_next_index])], + oper, + target, + ) if matches == look_for_match: return matches return not look_for_match @@ -138,7 +175,17 @@ class DbMemory(DbBase): for k, v in q_filter.items(): k_list = k.split(".") operator = "eq" - if k_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): + if k_list[-1] in ( + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "cont", + "ncont", + "neq", + ): operator = k_list.pop() matches = recursive_find(k_list, 0, row, operator, v) if not matches: @@ -199,11 +246,18 @@ class DbMemory(DbBase): if not fail_on_more: return deepcopy(row) if result: - raise DbException("Found more than one entry with filter='{}'".format(q_filter), - HTTPStatus.CONFLICT.value) + raise DbException( + "Found more than one entry with filter='{}'".format( + q_filter + ), + HTTPStatus.CONFLICT.value, + ) result = row if not result and fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + raise DbException( + "Not found entry with filter='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) return deepcopy(result) except Exception as e: # TODO refine raise DbException(str(e)) @@ -244,14 +298,26 @@ class DbMemory(DbBase): break else: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + raise DbException( + "Not found entry with filter='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) return None del self.db[table][i] return {"deleted": 1} except Exception as e: # TODO refine raise DbException(str(e)) - def _update(self, db_item, update_dict, unset=None, pull=None, push=None): + def _update( + self, + db_item, + update_dict, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): """ Modifies an entry at database :param db_item: entry of the table to update @@ -260,10 +326,14 @@ class DbMemory(DbBase): ignored. If not exist, it is ignored :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: True if database has been changed, False if not; Exception on error """ + def _iterate_keys(k, db_nested, populate=True): k_list = k.split(".") k_item_prev = k_list[0] @@ -275,7 +345,9 @@ class DbMemory(DbBase): if isinstance(db_nested[k_item_prev], dict): if k_item not in db_nested[k_item_prev]: if not populate: - raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + raise DbException( + "Cannot set '{}', not existing '{}'".format(k, k_item) + ) populated = True db_nested[k_item_prev][k_item] = None elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): @@ -283,17 +355,28 @@ class DbMemory(DbBase): k_item = int(k_item) if k_item >= len(db_nested[k_item_prev]): if not populate: - raise DbException("Cannot set '{}', index too large '{}'".format(k, k_item)) + raise DbException( + "Cannot set '{}', index too large '{}'".format( + k, k_item + ) + ) populated = True - db_nested[k_item_prev] += [None] * (k_item - len(db_nested[k_item_prev]) + 1) + db_nested[k_item_prev] += [None] * ( + k_item - len(db_nested[k_item_prev]) + 1 + ) elif db_nested[k_item_prev] is None: if not populate: - raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + raise DbException( + "Cannot set '{}', not existing '{}'".format(k, k_item) + ) populated = True db_nested[k_item_prev] = {k_item: None} else: # number, string, boolean, ... or list but with not integer key - raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_item_prev, - db_nested[k_item_prev])) + raise DbException( + "Cannot set '{}' on existing '{}={}'".format( + k, k_item_prev, db_nested[k_item_prev] + ) + ) db_nested = db_nested[k_item_prev] k_item_prev = k_item return db_nested, k_item_prev, populated @@ -308,7 +391,9 @@ class DbMemory(DbBase): if unset: for dot_k in unset: try: - dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) del dict_to_update[key_to_update] updated = True except Exception: @@ -316,30 +401,95 @@ class DbMemory(DbBase): if pull: for dot_k, v in pull.items(): try: - dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) except Exception: continue if key_to_update not in dict_to_update: continue if not isinstance(dict_to_update[key_to_update], list): - raise DbException("Cannot pull '{}'. Target is not a list".format(dot_k)) + raise DbException( + "Cannot pull '{}'. Target is not a list".format(dot_k) + ) while v in dict_to_update[key_to_update]: dict_to_update[key_to_update].remove(v) updated = True + if pull_list: + for dot_k, v in pull_list.items(): + if not isinstance(v, list): + raise DbException( + "Invalid content at pull_list, '{}' must be an array".format( + dot_k + ), + http_code=HTTPStatus.BAD_REQUEST, + ) + try: + dict_to_update, key_to_update, _ = _iterate_keys( + dot_k, db_item, populate=False + ) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot pull_list '{}'. Target is not a list".format(dot_k) + ) + for single_v in v: + while single_v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(single_v) + updated = True if push: for dot_k, v in push.items(): - dict_to_update, key_to_update, populated = _iterate_keys(dot_k, db_item) - if isinstance(dict_to_update, dict) and key_to_update not in dict_to_update: + dict_to_update, key_to_update, populated = _iterate_keys( + dot_k, db_item + ) + if ( + isinstance(dict_to_update, dict) + and key_to_update not in dict_to_update + ): dict_to_update[key_to_update] = [v] updated = True elif populated and dict_to_update[key_to_update] is None: dict_to_update[key_to_update] = [v] updated = True elif not isinstance(dict_to_update[key_to_update], list): - raise DbException("Cannot push '{}'. Target is not a list".format(dot_k)) + raise DbException( + "Cannot push '{}'. Target is not a list".format(dot_k) + ) else: dict_to_update[key_to_update].append(v) updated = True + if push_list: + for dot_k, v in push_list.items(): + if not isinstance(v, list): + raise DbException( + "Invalid content at push_list, '{}' must be an array".format( + dot_k + ), + http_code=HTTPStatus.BAD_REQUEST, + ) + dict_to_update, key_to_update, populated = _iterate_keys( + dot_k, db_item + ) + if ( + isinstance(dict_to_update, dict) + and key_to_update not in dict_to_update + ): + dict_to_update[key_to_update] = v.copy() + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = v.copy() + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException( + "Cannot push '{}'. Target is not a list".format(dot_k), + http_code=HTTPStatus.CONFLICT, + ) + else: + dict_to_update[key_to_update] += v + updated = True return updated except DbException: @@ -347,7 +497,18 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def set_one( + self, + table, + q_filter, + update_dict, + fail_on_empty=True, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): """ Modifies an entry at database :param table: collection or table @@ -359,28 +520,63 @@ class DbMemory(DbBase): ignored. If not exist, it is ignored :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified. None if no matching is found. """ with self.lock: for i, db_item in self._find(table, self._format_filter(q_filter)): - updated = self._update(db_item, update_dict, unset=unset, pull=pull, push=push) + updated = self._update( + db_item, + update_dict, + unset=unset, + pull=pull, + push=push, + push_list=push_list, + pull_list=pull_list, + ) return {"updated": 1 if updated else 0} else: if fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + raise DbException( + "Not found entry with _id='{}'".format(q_filter), + HTTPStatus.NOT_FOUND, + ) return None - def set_list(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def set_list( + self, + table, + q_filter, + update_dict, + unset=None, + pull=None, + push=None, + push_list=None, + pull_list=None, + ): + """Modifies al matching entries at database. Same as push. Do not fail if nothing matches""" with self.lock: updated = 0 - for i, db_item in self._find(table, self._format_filter(q_filter)): - if self._update(db_item, update_dict, unset=unset, pull=pull, push=push): + found = 0 + for _, db_item in self._find(table, self._format_filter(q_filter)): + found += 1 + if self._update( + db_item, + update_dict, + unset=unset, + pull=pull, + push=push, + push_list=push_list, + pull_list=pull_list, + ): updated += 1 - if i == 0 and fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) - return {"updated": updated} if i else None + # if not found and fail_on_empty: + # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return {"updated": updated} if found else None def replace(self, table, _id, indata, fail_on_empty=True): """ @@ -398,7 +594,10 @@ class DbMemory(DbBase): break else: if fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(_id), HTTPStatus.NOT_FOUND) + raise DbException( + "Not found entry with _id='{}'".format(_id), + HTTPStatus.NOT_FOUND, + ) return None self.db[table][i] = deepcopy(indata) return {"updated": 1} @@ -412,7 +611,7 @@ class DbMemory(DbBase): Add a new entry at database :param table: collection or table :param indata: content to be added - :return: database id of the inserted element. Raises a DbException on error + :return: database '_id' of the inserted element. Raises a DbException on error """ try: id = indata.get("_id") @@ -432,7 +631,7 @@ class DbMemory(DbBase): Add a new entry at database :param table: collection or table :param indata_list: list content to be added - :return: database ids of the inserted element. Raises a DbException on error + :return: list of inserted 'id's. Raises a DbException on error """ try: _ids = [] @@ -452,7 +651,7 @@ class DbMemory(DbBase): raise DbException(str(e)) -if __name__ == '__main__': +if __name__ == "__main__": # some test code db = DbMemory() db.create("test", {"_id": 1, "data": 1})