From: tierno Date: Thu, 19 Sep 2019 09:23:44 +0000 (+0000) Subject: fix in dbmemory query using '.cont' and '.ncont' X-Git-Tag: v7.0.0rc1~4 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fcommon.git;a=commitdiff_plain;h=40e326a114e66596d62480f9db59d142e17bac0b fix in dbmemory query using '.cont' and '.ncont' Change-Id: I12404db1eec2bb68f72fde6dadf884ac949bf30d Signed-off-by: tierno --- diff --git a/osm_common/__init__.py b/osm_common/__init__.py index bf77d1a..6fc66a5 100644 --- a/osm_common/__init__.py +++ b/osm_common/__init__.py @@ -15,5 +15,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version = '6.0.2.post1' -date_version = '2019-08-28' +version = '6.0.2.post3' +date_version = '2019-09-19' diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index d12d03d..1b6d7ac 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -62,72 +62,76 @@ class DbMemory(DbBase): def _find(self, table, q_filter): - def recursive_find(key_list, key_next_index, content, operator, target): + def recursive_find(key_list, key_next_index, content, oper, target): if key_next_index == len(key_list) or content is None: try: - if operator == "eq": - if isinstance(target, list) and not isinstance(content, list): - return True if content in target else False - return True if content == target else False - elif operator in ("neq", "ne"): - if isinstance(target, list) and not isinstance(content, list): - return True if content not in target else False - return True if content != target else False - if operator == "gt": + if oper in ("eq", "cont"): + if isinstance(target, list): + if isinstance(content, list): + return any(content_item in target for content_item in content) + return content in target + elif isinstance(content, list): + return target in content + else: + return content == target + elif oper in ("neq", "ne", "ncont"): + if isinstance(target, list): + if isinstance(content, list): + return all(content_item not in target for content_item in content) + return content not in target + elif isinstance(content, list): + return target not in content + else: + return content != target + if oper == "gt": return content > target - elif operator == "gte": + elif oper == "gte": return content >= target - elif operator == "lt": + elif oper == "lt": return content < target - elif operator == "lte": + elif oper == "lte": return content <= target - elif operator == "cont": - return content in target - elif operator == "ncont": - return content not in target else: raise DbException("Unknown filter operator '{}' in key '{}'". - format(operator, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST) + format(oper, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST) except TypeError: return False elif isinstance(content, dict): - return recursive_find(key_list, key_next_index+1, content.get(key_list[key_next_index]), operator, + return recursive_find(key_list, key_next_index + 1, content.get(key_list[key_next_index]), oper, target) elif isinstance(content, list): look_for_match = True # when there is a match return immediately - if (target is None and operator not in ("neq", "ne")) or \ - (target is not None and operator in ("neq", "ne")): - look_for_match = False # when there is a match return immediately + if (target is None) != (oper in ("neq", "ne", "ncont")): # one True and other False (Xor) + look_for_match = False # when there is not a match return immediately for content_item in content: if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict): + matches = True for k2, v2 in target.items(): k_new_list = k2.split(".") new_operator = "eq" if k_new_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): new_operator = k_new_list.pop() if not recursive_find(k_new_list, 0, content_item, new_operator, v2): - match = False + matches = False break - else: - match = True else: - match = recursive_find(key_list, key_next_index, content_item, operator, target) - if match == look_for_match: - return match + matches = recursive_find(key_list, key_next_index, content_item, oper, target) + if matches == look_for_match: + return matches if key_list[key_next_index].isdecimal() and int(key_list[key_next_index]) < len(content): - match = recursive_find(key_list, key_next_index+1, content[int(key_list[key_next_index])], - operator, target) - if match == look_for_match: - return match + matches = recursive_find(key_list, key_next_index + 1, content[int(key_list[key_next_index])], + oper, target) + if matches == look_for_match: + return matches return not look_for_match else: # content is not dict, nor list neither None, so not found - if operator in ("neq", "ne"): - return True if target is None else False + if oper in ("neq", "ne", "ncont"): + return target is not None else: - return True if target is None else False + return target is None for i, row in enumerate(self.db.get(table, ())): q_filter = q_filter or {} @@ -136,8 +140,8 @@ class DbMemory(DbBase): operator = "eq" if k_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"): operator = k_list.pop() - match = recursive_find(k_list, 0, row, operator, v) - if not match: + matches = recursive_find(k_list, 0, row, operator, v) + if not matches: break else: # match @@ -274,26 +278,26 @@ class DbMemory(DbBase): for k, v in update_dict.items(): db_nested = db_item k_list = k.split(".") - k_nested_prev = k_list[0] - for k_nested in k_list[1:]: - if isinstance(db_nested[k_nested_prev], dict): - if k_nested not in db_nested[k_nested_prev]: - db_nested[k_nested_prev][k_nested] = None - elif isinstance(db_nested[k_nested_prev], list) and k_nested.isdigit(): + k_item_prev = k_list[0] + for k_item in k_list[1:]: + if isinstance(db_nested[k_item_prev], dict): + if k_item not in db_nested[k_item_prev]: + db_nested[k_item_prev][k_item] = None + elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit(): # extend list with Nones if index greater than list - k_nested = int(k_nested) - if k_nested >= len(db_nested[k_nested_prev]): - db_nested[k_nested_prev] += [None] * (k_nested - len(db_nested[k_nested_prev]) + 1) - elif db_nested[k_nested_prev] is None: - db_nested[k_nested_prev] = {k_nested: None} + k_item = int(k_item) + if k_item >= len(db_nested[k_item_prev]): + db_nested[k_item_prev] += [None] * (k_item - len(db_nested[k_item_prev]) + 1) + elif db_nested[k_item_prev] is None: + db_nested[k_item_prev] = {k_item: None} else: # number, string, boolean, ... or list but with not integer key - raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_nested_prev, - db_nested[k_nested_prev])) + raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_item_prev, + db_nested[k_item_prev])) - db_nested = db_nested[k_nested_prev] - k_nested_prev = k_nested + db_nested = db_nested[k_item_prev] + k_item_prev = k_item - db_nested[k_nested_prev] = v + db_nested[k_item_prev] = v return {"updated": 1} except DbException: raise @@ -354,16 +358,17 @@ class DbMemory(DbBase): """ try: _ids = [] - for indata in indata_list: - _id = indata.get("_id") - if not _id: - _id = str(uuid4()) - indata["_id"] = _id - with self.lock: - if table not in self.db: - self.db[table] = [] - self.db[table].append(deepcopy(indata)) - _ids.append(_id) + with self.lock: + for indata in indata_list: + _id = indata.get("_id") + if not _id: + _id = str(uuid4()) + indata["_id"] = _id + with self.lock: + if table not in self.db: + self.db[table] = [] + self.db[table].append(deepcopy(indata)) + _ids.append(_id) return _ids except Exception as e: # TODO refine raise DbException(str(e)) diff --git a/osm_common/tests/test_dbmemory.py b/osm_common/tests/test_dbmemory.py index 552e8f1..b3b20ff 100644 --- a/osm_common/tests/test_dbmemory.py +++ b/osm_common/tests/test_dbmemory.py @@ -53,7 +53,7 @@ def db_memory_with_many_data(request): db.create_list("test", [ {"_id": 1, "data": {"data2": {"data3": 1}}, "list": [{"a": 1}], "text": "sometext"}, - {"_id": 2, "data": {"data2": {"data3": 2}}, "list": [{"a": 2}]}, + {"_id": 2, "data": {"data2": {"data3": 2}}, "list": [{"a": 2}], "list2": [1, 2, 3]}, {"_id": 3, "data": {"data2": {"data3": 3}}, "list": [{"a": 3}]}, {"_id": 4, "data": {"data2": {"data3": 4}}, "list": [{"a": 4}, {"a": 0}]}, {"_id": 5, "data": {"data2": {"data3": 5}}, "list": [{"a": 5}]}, @@ -176,7 +176,11 @@ def test_get_one(db_memory_with_data, table, db_filter, expected_data): ({"_id": 1}, [1]), ({"data.data2.data3": 2}, [2]), ({"data.data2.data3.eq": 2}, [2]), + ({"data.data2.data3": [2]}, [2]), + ({"data.data2.data3.cont": [2]}, [2]), ({"data.data2.data3.neq": 2}, [1, 3, 4, 5, 6, 7, 8]), + ({"data.data2.data3.neq": [2]}, [1, 3, 4, 5, 6, 7, 8]), + ({"data.data2.data3.ncont": [2]}, [1, 3, 4, 5, 6, 7, 8]), ({"data.data2.data3": [2, 3]}, [2, 3]), ({"data.data2.data3.gt": 4}, [5, 6, 7]), ({"data.data2.data3.gte": 4}, [4, 5, 6, 7]), @@ -184,6 +188,10 @@ def test_get_one(db_memory_with_data, table, db_filter, expected_data): ({"data.data2.data3.lte": 4}, [1, 2, 3, 4]), ({"data.data2.data3.lte": 4.5}, [1, 2, 3, 4]), ({"data.data2.data3.gt": "text"}, []), + ({"nonexist.nonexist": "4"}, []), + ({"nonexist.nonexist": None}, [1, 2, 3, 4, 5, 6, 7, 8]), + ({"nonexist.nonexist.neq": "4"}, [1, 2, 3, 4, 5, 6, 7, 8]), + ({"nonexist.nonexist.neq": None}, []), ({"text.eq": "sometext"}, [1]), ({"text.neq": "sometext"}, [2, 3, 4, 5, 6, 7, 8]), ({"text.eq": "somet"}, []), @@ -198,6 +206,10 @@ def test_get_one(db_memory_with_data, table, db_filter, expected_data): ({"data.data2.data3": None}, [8]), ({"data.data2": "4"}, []), ({"list.0.a": 1}, [1, 6]), + ({"list2": 1}, [2]), + ({"list2": [1, 5]}, [2]), + ({"list2": [1, 2]}, [2]), + ({"list2": [5, 7]}, []), ({"list.ANYINDEX.a": 1}, [1]), ({"list.a": 3, "list.b": 1}, [8]), ({"list.ANYINDEX.a": 3, "list.ANYINDEX.b": 1}, []),