From 7fc50dd812c4006d1b34d52e3be0de79528724ba Mon Sep 17 00:00:00 2001 From: tierno Date: Mon, 17 Feb 2020 12:01:38 +0000 Subject: [PATCH] adding dbmemory set_list and tests Change-Id: Iaf9465fb0bae5e12f19a2810b454209ef39614c3 Signed-off-by: tierno --- osm_common/dbmemory.py | 145 ++++++++++++++++++++++-------- osm_common/tests/test_dbmemory.py | 71 +++++++++++++++ tox.ini | 2 +- 3 files changed, 182 insertions(+), 36 deletions(-) diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index 1b6d7ac..bfc396d 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -251,6 +251,99 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) + def _update(self, db_item, update_dict, unset=None, pull=None, push=None): + """ + Modifies an entry at database + :param db_item: entry of the table to update + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is + ignored. If not exist, it is ignored + :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value + if exist in the array is removed. If not exist, it is ignored + :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value + is appended to the end of the array + :return: True if database has been changed, False if not; Exception on error + """ + def _iterate_keys(k, db_nested, populate=True): + k_list = k.split(".") + k_item_prev = k_list[0] + populated = False + for k_item in k_list[1:]: + if isinstance(db_nested[k_item_prev], dict): + if k_item not in db_nested[k_item_prev]: + if not populate: + raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev][k_item] = None + elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): + # extend list with Nones if index greater than list + k_item = int(k_item) + if k_item >= len(db_nested[k_item_prev]): + if not populate: + raise DbException("Cannot set '{}', index too large '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev] += [None] * (k_item - len(db_nested[k_item_prev]) + 1) + elif db_nested[k_item_prev] is None: + if not populate: + raise DbException("Cannot set '{}', not existing '{}'".format(k, k_item)) + populated = True + db_nested[k_item_prev] = {k_item: None} + else: # number, string, boolean, ... or list but with not integer key + raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_item_prev, + db_nested[k_item_prev])) + db_nested = db_nested[k_item_prev] + k_item_prev = k_item + return db_nested, k_item_prev, populated + + updated = False + try: + if update_dict: + for dot_k, v in update_dict.items(): + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item) + dict_to_update[key_to_update] = v + updated = True + if unset: + for dot_k in unset: + try: + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + del dict_to_update[key_to_update] + updated = True + except Exception: + pass + if pull: + for dot_k, v in pull.items(): + try: + dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item, populate=False) + except Exception: + continue + if key_to_update not in dict_to_update: + continue + if not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot pull '{}'. Target is not a list".format(dot_k)) + while v in dict_to_update[key_to_update]: + dict_to_update[key_to_update].remove(v) + updated = True + if push: + for dot_k, v in push.items(): + dict_to_update, key_to_update, populated = _iterate_keys(dot_k, db_item) + if isinstance(dict_to_update, dict) and key_to_update not in dict_to_update: + dict_to_update[key_to_update] = [v] + updated = True + elif populated and dict_to_update[key_to_update] is None: + dict_to_update[key_to_update] = [v] + updated = True + elif not isinstance(dict_to_update[key_to_update], list): + raise DbException("Cannot push '{}'. Target is not a list".format(dot_k)) + else: + dict_to_update[key_to_update].append(v) + updated = True + + return updated + except DbException: + raise + except Exception as e: # TODO refine + raise DbException(str(e)) + def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): """ Modifies an entry at database @@ -267,42 +360,24 @@ class DbMemory(DbBase): is appended to the end of the array :return: Dict with the number of entries modified. None if no matching is found. """ - try: - with self.lock: - for i, db_item in self._find(table, self._format_filter(q_filter)): - break - else: - if fail_on_empty: - raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) - return None - for k, v in update_dict.items(): - db_nested = db_item - k_list = k.split(".") - k_item_prev = k_list[0] - for k_item in k_list[1:]: - if isinstance(db_nested[k_item_prev], dict): - if k_item not in db_nested[k_item_prev]: - db_nested[k_item_prev][k_item] = None - elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): - # extend list with Nones if index greater than list - k_item = int(k_item) - if k_item >= len(db_nested[k_item_prev]): - db_nested[k_item_prev] += [None] * (k_item - len(db_nested[k_item_prev]) + 1) - elif db_nested[k_item_prev] is None: - db_nested[k_item_prev] = {k_item: None} - else: # number, string, boolean, ... or list but with not integer key - raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_item_prev, - db_nested[k_item_prev])) - - db_nested = db_nested[k_item_prev] - k_item_prev = k_item + with self.lock: + for i, db_item in self._find(table, self._format_filter(q_filter)): + updated = self._update(db_item, update_dict, unset=unset, pull=pull, push=push) + return {"updated": 1 if updated else 0} + else: + if fail_on_empty: + raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return None - db_nested[k_item_prev] = v - return {"updated": 1} - except DbException: - raise - except Exception as e: # TODO refine - raise DbException(str(e)) + def set_list(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None): + with self.lock: + updated = 0 + for i, db_item in self._find(table, self._format_filter(q_filter)): + if self._update(db_item, update_dict, unset=unset, pull=pull, push=push): + updated += 1 + if i == 0 and fail_on_empty: + raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND) + return {"updated": updated} if i else None def replace(self, table, _id, indata, fail_on_empty=True): """ diff --git a/osm_common/tests/test_dbmemory.py b/osm_common/tests/test_dbmemory.py index b3b20ff..d2fbf4e 100644 --- a/osm_common/tests/test_dbmemory.py +++ b/osm_common/tests/test_dbmemory.py @@ -26,6 +26,7 @@ from unittest.mock import Mock from unittest.mock import MagicMock from osm_common.dbbase import DbException from osm_common.dbmemory import DbMemory +from copy import deepcopy __author__ = 'Eduardo Sousa ' @@ -720,3 +721,73 @@ class TestDbMemory(unittest.TestCase): else: db_men.set_one("table", {}, update_dict) self.assertEqual(db_content, expected, message) + + def test_set_one_pull(self): + example = {"a": [1, "1", 1], "d": {}, "n": None} + test_set = ( + # (database content, set-content, expected database content (None=fails), message) + (example, {"a": "1"}, {"a": [1, 1], "d": {}, "n": None}, "pull one item"), + (example, {"a": 1}, {"a": ["1"], "d": {}, "n": None}, "pull two items"), + (example, {"a": "v"}, example, "pull non existing item"), + (example, {"a.6": 1}, example, "pull non existing arrray"), + (example, {"d.b.c": 1}, example, "pull non existing arrray2"), + (example, {"b": 1}, example, "pull non existing arrray3"), + (example, {"d": 1}, None, "pull over dict"), + (example, {"n": 1}, None, "pull over None"), + ) + db_men = DbMemory() + db_men._find = Mock() + for db_content, pull_dict, expected, message in test_set: + db_content = deepcopy(db_content) + db_men._find.return_value = ((0, db_content), ) + if expected is None: + self.assertRaises(DbException, db_men.set_one, "table", {}, None, fail_on_empty=False, pull=pull_dict) + else: + db_men.set_one("table", {}, None, pull=pull_dict) + self.assertEqual(db_content, expected, message) + + def test_set_one_push(self): + example = {"a": [1, "1", 1], "d": {}, "n": None} + test_set = ( + # (database content, set-content, expected database content (None=fails), message) + (example, {"d.b.c": 1}, {"a": [1, "1", 1], "d": {"b": {"c": [1]}}, "n": None}, "push non existing arrray2"), + (example, {"b": 1}, {"a": [1, "1", 1], "d": {}, "b": [1], "n": None}, "push non existing arrray3"), + (example, {"a.6": 1}, {"a": [1, "1", 1, None, None, None, [1]], "d": {}, "n": None}, + "push non existing arrray"), + (example, {"a": 2}, {"a": [1, "1", 1, 2], "d": {}, "n": None}, "push one item"), + (example, {"a": {1: 1}}, {"a": [1, "1", 1, {1: 1}], "d": {}, "n": None}, "push a dict"), + (example, {"d": 1}, None, "push over dict"), + (example, {"n": 1}, None, "push over None"), + ) + db_men = DbMemory() + db_men._find = Mock() + for db_content, push_dict, expected, message in test_set: + db_content = deepcopy(db_content) + db_men._find.return_value = ((0, db_content), ) + if expected is None: + self.assertRaises(DbException, db_men.set_one, "table", {}, None, fail_on_empty=False, push=push_dict) + else: + db_men.set_one("table", {}, None, push=push_dict) + self.assertEqual(db_content, expected, message) + + def test_unset_one(self): + example = {"a": [1, "1", 1], "d": {}, "n": None} + test_set = ( + # (database content, set-content, expected database content (None=fails), message) + (example, {"d.b.c": 1}, example, "unset non existing"), + (example, {"b": 1}, example, "unset non existing"), + (example, {"a.6": 1}, example, "unset non existing arrray"), + (example, {"a": 2}, {"d": {}, "n": None}, "unset array"), + (example, {"d": 1}, {"a": [1, "1", 1], "n": None}, "unset dict"), + (example, {"n": 1}, {"a": [1, "1", 1], "d": {}}, "unset None"), + ) + db_men = DbMemory() + db_men._find = Mock() + for db_content, unset_dict, expected, message in test_set: + db_content = deepcopy(db_content) + db_men._find.return_value = ((0, db_content), ) + if expected is None: + self.assertRaises(DbException, db_men.set_one, "table", {}, None, fail_on_empty=False, unset=unset_dict) + else: + db_men.set_one("table", {}, None, unset=unset_dict) + self.assertEqual(db_content, expected, message) diff --git a/tox.ini b/tox.ini index 1cf3d54..7200455 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ commands = basepython = python3 deps = pycrypto pytest -commands = python3 -m unittest osm_common.tests.test_dbbase +commands = python3 -m unittest discover osm_common.tests [testenv:build] -- 2.25.1