From 399f6c3acc2e8852d463844a562bcde063c912e2 Mon Sep 17 00:00:00 2001 From: tierno Date: Tue, 12 May 2020 07:36:41 +0000 Subject: [PATCH] allow dabase update with pushlist It can push several items at same time Change-Id: I5ffbcb0c112a0c539862901ec5659ffca768316d Signed-off-by: tierno --- osm_common/dbbase.py | 9 ++++++-- osm_common/dbmemory.py | 34 ++++++++++++++++++++++++++----- osm_common/dbmongo.py | 25 +++++++++++++++-------- osm_common/tests/test_dbmemory.py | 27 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/osm_common/dbbase.py b/osm_common/dbbase.py index 5f05e0d..c71c5e1 100644 --- a/osm_common/dbbase.py +++ b/osm_common/dbbase.py @@ -151,7 +151,8 @@ class DbBase(object): """ raise DbException("Method 'create_list' not implemented") - def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None, + push_list=None): """ Modifies an entry at database :param table: collection or table @@ -165,11 +166,13 @@ class DbBase(object): if exist in the array is removed. If not exist, it is ignored :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified. None if no matching is found. """ raise DbException("Method 'set_one' not implemented") - def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None): + def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None, push_list=None): """ Modifies al matching entries at database :param table: collection or table @@ -181,6 +184,8 @@ class DbBase(object): if exist in the array is removed. If not exist, it is ignored :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified """ raise DbException("Method 'set_list' not implemented") diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index 196d5d9..6ad93d6 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -251,7 +251,7 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def _update(self, db_item, update_dict, unset=None, pull=None, push=None): + def _update(self, db_item, update_dict, unset=None, pull=None, push=None, push_list=None): """ Modifies an entry at database :param db_item: entry of the table to update @@ -262,6 +262,8 @@ class DbMemory(DbBase): if exist in the array is removed. If not exist, it is ignored :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: True if database has been changed, False if not; Exception on error """ def _iterate_keys(k, db_nested, populate=True): @@ -340,6 +342,24 @@ class DbMemory(DbBase): else: dict_to_update[key_to_update].append(v) updated = True + if push_list: + for dot_k, v in push_list.items(): + if not isinstance(v, list): + raise DbException("Invalid content at push_list, '{}' must be an array".format(dot_k), + http_code=HTTPStatus.BAD_REQUEST) + dict_to_update, key_to_update, populated = _iterate_keys(dot_k, db_item) + if isinstance(dict_to_update, dict) and key_to_update not in dict_to_update: + dict_to_update[key_to_update] = v.copy() + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = v.copy() + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot push '{}'. Target is not a list".format(dot_k), + http_code=HTTPStatus.CONFLICT) + else: + dict_to_update[key_to_update] += v + updated = True return updated except DbException: @@ -347,7 +367,8 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None, + push_list=None): """ Modifies an entry at database :param table: collection or table @@ -361,24 +382,27 @@ class DbMemory(DbBase): if exist in the array is removed. If not exist, it is ignored :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified. None if no matching is found. """ with self.lock: for i, db_item in self._find(table, self._format_filter(q_filter)): - updated = self._update(db_item, update_dict, unset=unset, pull=pull, push=push) + updated = self._update(db_item, update_dict, unset=unset, pull=pull, push=push, push_list=push_list) return {"updated": 1 if updated else 0} else: if fail_on_empty: raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) return None - def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None): + def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None, push_list=None): + """Modifies al matching entries at database. Same as push. Do not fail if nothing matches""" with self.lock: updated = 0 found = 0 for _, db_item in self._find(table, self._format_filter(q_filter)): found += 1 - if self._update(db_item, update_dict, unset=unset, pull=pull, push=push): + if self._update(db_item, update_dict, unset=unset, pull=pull, push=push, push_list=push_list): updated += 1 # if not found and fail_on_empty: # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND) diff --git a/osm_common/dbmongo.py b/osm_common/dbmongo.py index e40d0e4..0784985 100644 --- a/osm_common/dbmongo.py +++ b/osm_common/dbmongo.py @@ -362,7 +362,8 @@ class DbMongo(DbBase): except Exception as e: # TODO refine raise DbException(e) - def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None, + push_list=None): """ Modifies an entry at database :param table: collection or table @@ -376,6 +377,8 @@ class DbMongo(DbBase): if exist in the array is removed. If not exist, it is ignored :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified. None if no matching is found. """ try: @@ -386,8 +389,10 @@ class DbMongo(DbBase): db_oper["$unset"] = unset if pull: db_oper["$pull"] = pull - if push: - db_oper["$push"] = push + if push or push_list: + db_oper["$push"] = push or {} + if push_list: + db_oper["$push"].update({k: {"$each": v} for k, v in push_list.items()}) with self.lock: collection = self.db[table] @@ -401,7 +406,7 @@ class DbMongo(DbBase): except Exception as e: # TODO refine raise DbException(e) - def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None): + def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None, push_list=None): """ Modifies al matching entries at database :param table: collection or table @@ -411,8 +416,10 @@ class DbMongo(DbBase): ignored. If not exist, it is ignored :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value if exist in the array is removed. If not exist, it is ignored - :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value - is appended to the end of the array + :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys, the + single value is appended to the end of the array + :param push_list: Same as push but values are arrays where each item is and appended instead of appending the + whole array :return: Dict with the number of entries modified """ try: @@ -423,8 +430,10 @@ class DbMongo(DbBase): db_oper["$unset"] = unset if pull: db_oper["$pull"] = pull - if push: - db_oper["$push"] = push + if push or push_list: + db_oper["$push"] = push or {} + if push_list: + db_oper["$push"].update({k: {"$each": v} for k, v in push_list.items()}) with self.lock: collection = self.db[table] rows = collection.update_many(self._format_filter(q_filter), db_oper) diff --git a/osm_common/tests/test_dbmemory.py b/osm_common/tests/test_dbmemory.py index cbc9715..344d3dd 100644 --- a/osm_common/tests/test_dbmemory.py +++ b/osm_common/tests/test_dbmemory.py @@ -771,6 +771,33 @@ class TestDbMemory(unittest.TestCase): db_men.set_one("table", {}, None, push=push_dict) self.assertEqual(db_content, expected, message) + def test_set_one_push_list(self): + example = {"a": [1, "1", 1], "d": {}, "n": None} + test_set = ( + # (database content, set-content, expected database content (None=fails), message) + (example, {"d.b.c": [1]}, {"a": [1, "1", 1], "d": {"b": {"c": [1]}}, "n": None}, + "push non existing arrray2"), + (example, {"b": [1]}, {"a": [1, "1", 1], "d": {}, "b": [1], "n": None}, "push non existing arrray3"), + (example, {"a.6": [1]}, {"a": [1, "1", 1, None, None, None, [1]], "d": {}, "n": None}, + "push non existing arrray"), + (example, {"a": [2, 3]}, {"a": [1, "1", 1, 2, 3], "d": {}, "n": None}, "push two item"), + (example, {"a": [{1: 1}]}, {"a": [1, "1", 1, {1: 1}], "d": {}, "n": None}, "push a dict"), + (example, {"d": [1]}, None, "push over dict"), + (example, {"n": [1]}, None, "push over None"), + (example, {"a": 1}, None, "invalid push list non an array"), + ) + db_men = DbMemory() + db_men._find = Mock() + for db_content, push_list, expected, message in test_set: + db_content = deepcopy(db_content) + db_men._find.return_value = ((0, db_content), ) + if expected is None: + self.assertRaises(DbException, db_men.set_one, "table", {}, None, fail_on_empty=False, + push_list=push_list) + else: + db_men.set_one("table", {}, None, push_list=push_list) + self.assertEqual(db_content, expected, message) + def test_unset_one(self): example = {"a": [1, "1", 1], "d": {}, "n": None} test_set = ( -- 2.17.1