X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Fdbmemory.py;h=51ae810c0ea512c5ed9aa236facd51d6935cf17f;hb=4ce854c2cfcdddf4d049ee312182c65832b3f5d4;hp=d12d03ddc3a20ce52e3a18d907ddb920c0edde2f;hpb=ae049d8467e5ce1b1be8487ed93031b594dc0230;p=osm%2Fcommon.git diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index d12d03d..51ae810 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -62,72 +62,76 @@ class DbMemory(DbBase): def _find(self, table, q_filter): - def recursive_find(key_list, key_next_index, content, operator, target): + def recursive_find(key_list, key_next_index, content, oper, target): if key_next_index == len(key_list) or content is None: try: - if operator == "eq": - if isinstance(target, list) and not isinstance(content, list): - return True if content in target else False - return True if content == target else False - elif operator in ("neq", "ne"): - if isinstance(target, list) and not isinstance(content, list): - return True if content not in target else False - return True if content != target else False - if operator == "gt": + if oper in ("eq", "cont"): + if isinstance(target, list): + if isinstance(content, list): + return any(content_item in target for content_item in content) + return content in target + elif isinstance(content, list): + return target in content + else: + return content == target + elif oper in ("neq", "ne", "ncont"): + if isinstance(target, list): + if isinstance(content, list): + return all(content_item not in target for content_item in content) + return content not in target + elif isinstance(content, list): + return target not in content + else: + return content != target + if oper == "gt": return content > target - elif operator == "gte": + elif oper == "gte": return content >= target - elif operator == "lt": + elif oper == "lt": return content < target - elif operator == "lte": + elif oper == "lte": return content <= target - elif operator == "cont": - return content in target - elif operator == "ncont": - return content not in target else: raise DbException("Unknown filter operator '{}' in key '{}'". - format(operator, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST) + format(oper, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST) except TypeError: return False elif isinstance(content, dict): - return recursive_find(key_list, key_next_index+1, content.get(key_list[key_next_index]), operator, + return recursive_find(key_list, key_next_index + 1, content.get(key_list[key_next_index]), oper, target) elif isinstance(content, list): look_for_match = True # when there is a match return immediately - if (target is None and operator not in ("neq", "ne")) or \ - (target is not None and operator in ("neq", "ne")): - look_for_match = False # when there is a match return immediately + if (target is None) != (oper in ("neq", "ne", "ncont")): # one True and other False (Xor) + look_for_match = False # when there is not a match return immediately for content_item in content: if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict): + matches = True for k2, v2 in target.items(): k_new_list = k2.split(".") new_operator = "eq" if k_new_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): new_operator = k_new_list.pop() if not recursive_find(k_new_list, 0, content_item, new_operator, v2): - match = False + matches = False break - else: - match = True else: - match = recursive_find(key_list, key_next_index, content_item, operator, target) - if match == look_for_match: - return match + matches = recursive_find(key_list, key_next_index, content_item, oper, target) + if matches == look_for_match: + return matches if key_list[key_next_index].isdecimal() and int(key_list[key_next_index]) < len(content): - match = recursive_find(key_list, key_next_index+1, content[int(key_list[key_next_index])], - operator, target) - if match == look_for_match: - return match + matches = recursive_find(key_list, key_next_index + 1, content[int(key_list[key_next_index])], + oper, target) + if matches == look_for_match: + return matches return not look_for_match else: # content is not dict, nor list neither None, so not found - if operator in ("neq", "ne"): - return True if target is None else False + if oper in ("neq", "ne", "ncont"): + return target is not None else: - return True if target is None else False + return target is None for i, row in enumerate(self.db.get(table, ())): q_filter = q_filter or {} @@ -136,8 +140,8 @@ class DbMemory(DbBase): operator = "eq" if k_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): operator = k_list.pop() - match = recursive_find(k_list, 0, row, operator, v) - if not match: + matches = recursive_find(k_list, 0, row, operator, v) + if not matches: break else: # match @@ -247,59 +251,184 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def _update(self, db_item, update_dict, unset=None, pull=None, push=None, push_list=None, pull_list=None): """ Modifies an entry at database - :param table: collection or table - :param q_filter: Filter + :param db_item: entry of the table to update :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value - :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case - it raises a DbException :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is ignored. If not exist, it is ignored :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array - :return: Dict with the number of entries modified. None if no matching is found. + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array + :return: True if database has been changed, False if not; Exception on error """ - try: - with self.lock: - for i, db_item in self._find(table, self._format_filter(q_filter)): - break - else: - if fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) - return None - for k, v in update_dict.items(): - db_nested = db_item - k_list = k.split(".") - k_nested_prev = k_list[0] - for k_nested in k_list[1:]: - if isinstance(db_nested[k_nested_prev], dict): - if k_nested not in db_nested[k_nested_prev]: - db_nested[k_nested_prev][k_nested] = None - elif isinstance(db_nested[k_nested_prev], list) and k_nested.isdigit(): - # extend list with Nones if index greater than list - k_nested = int(k_nested) - if k_nested >= len(db_nested[k_nested_prev]): - db_nested[k_nested_prev] += [None] * (k_nested - len(db_nested[k_nested_prev]) + 1) - elif db_nested[k_nested_prev] is None: - db_nested[k_nested_prev] = {k_nested: None} - else: # number, string, boolean, ... or list but with not integer key - raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_nested_prev, - db_nested[k_nested_prev])) + def _iterate_keys(k, db_nested, populate=True): + k_list = k.split(".") + k_item_prev = k_list[0] + populated = False + if k_item_prev not in db_nested and populate: + populated = True + db_nested[k_item_prev] = None + for k_item in k_list[1:]: + if isinstance(db_nested[k_item_prev], dict): + if k_item not in db_nested[k_item_prev]: + if not populate: + raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev][k_item] = None + elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): + # extend list with Nones if index greater than list + k_item = int(k_item) + if k_item >= len(db_nested[k_item_prev]): + if not populate: + raise DbException("Cannot set '{}', index too large '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev] += [None] * (k_item - len(db_nested[k_item_prev]) + 1) + elif db_nested[k_item_prev] is None: + if not populate: + raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev] = {k_item: None} + else: # number, string, boolean, ... or list but with not integer key + raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_item_prev, + db_nested[k_item_prev])) + db_nested = db_nested[k_item_prev] + k_item_prev = k_item + return db_nested, k_item_prev, populated - db_nested = db_nested[k_nested_prev] - k_nested_prev = k_nested + updated = False + try: + if update_dict: + for dot_k, v in update_dict.items(): + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item) + dict_to_update[key_to_update] = v + updated = True + if unset: + for dot_k in unset: + try: + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + del dict_to_update[key_to_update] + updated = True + except Exception: + pass + if pull: + for dot_k, v in pull.items(): + try: + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot pull '{}'. Target is not a list".format(dot_k)) + while v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(v) + updated = True + if pull_list: + for dot_k, v in pull_list.items(): + if not isinstance(v, list): + raise DbException("Invalid content at pull_list, '{}' must be an array".format(dot_k), + http_code=HTTPStatus.BAD_REQUEST) + try: + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot pull_list '{}'. Target is not a list".format(dot_k)) + for single_v in v: + while single_v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(single_v) + updated = True + if push: + for dot_k, v in push.items(): + dict_to_update, key_to_update, populated = _iterate_keys(dot_k, db_item) + if isinstance(dict_to_update, dict) and key_to_update not in dict_to_update: + dict_to_update[key_to_update] = [v] + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = [v] + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot push '{}'. Target is not a list".format(dot_k)) + else: + dict_to_update[key_to_update].append(v) + updated = True + if push_list: + for dot_k, v in push_list.items(): + if not isinstance(v, list): + raise DbException("Invalid content at push_list, '{}' must be an array".format(dot_k), + http_code=HTTPStatus.BAD_REQUEST) + dict_to_update, key_to_update, populated = _iterate_keys(dot_k, db_item) + if isinstance(dict_to_update, dict) and key_to_update not in dict_to_update: + dict_to_update[key_to_update] = v.copy() + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = v.copy() + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot push '{}'. Target is not a list".format(dot_k), + http_code=HTTPStatus.CONFLICT) + else: + dict_to_update[key_to_update] += v + updated = True - db_nested[k_nested_prev] = v - return {"updated": 1} + return updated except DbException: raise except Exception as e: # TODO refine raise DbException(str(e)) + def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None, + push_list=None, pull_list=None): + """ + Modifies an entry at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is + ignored. If not exist, it is ignored + :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value + if exist in the array is removed. If not exist, it is ignored + :param pull_list: Same as pull but values are arrays where each item is removed from the array + :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value + is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array + :return: Dict with the number of entries modified. None if no matching is found. + """ + with self.lock: + for i, db_item in self._find(table, self._format_filter(q_filter)): + updated = self._update(db_item, update_dict, unset=unset, pull=pull, push=push, push_list=push_list, + pull_list=pull_list) + return {"updated": 1 if updated else 0} + else: + if fail_on_empty: + raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return None + + def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None, push_list=None, pull_list=None): + """Modifies al matching entries at database. Same as push. Do not fail if nothing matches""" + with self.lock: + updated = 0 + found = 0 + for _, db_item in self._find(table, self._format_filter(q_filter)): + found += 1 + if self._update(db_item, update_dict, unset=unset, pull=pull, push=push, push_list=push_list, + pull_list=pull_list): + updated += 1 + # if not found and fail_on_empty: + # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return {"updated": updated} if found else None + def replace(self, table, _id, indata, fail_on_empty=True): """ Replace the content of an entry @@ -330,7 +459,7 @@ class DbMemory(DbBase): Add a new entry at database :param table: collection or table :param indata: content to be added - :return: database id of the inserted element. Raises a DbException on error + :return: database '_id' of the inserted element. Raises a DbException on error """ try: id = indata.get("_id") @@ -350,20 +479,21 @@ class DbMemory(DbBase): Add a new entry at database :param table: collection or table :param indata_list: list content to be added - :return: database ids of the inserted element. Raises a DbException on error + :return: list of inserted 'id's. Raises a DbException on error """ try: _ids = [] - for indata in indata_list: - _id = indata.get("_id") - if not _id: - _id = str(uuid4()) - indata["_id"] = _id - with self.lock: - if table not in self.db: - self.db[table] = [] - self.db[table].append(deepcopy(indata)) - _ids.append(_id) + with self.lock: + for indata in indata_list: + _id = indata.get("_id") + if not _id: + _id = str(uuid4()) + indata["_id"] = _id + with self.lock: + if table not in self.db: + self.db[table] = [] + self.db[table].append(deepcopy(indata)) + _ids.append(_id) return _ids except Exception as e: # TODO refine raise DbException(str(e))