9f8c571e9ec5d6a6e45ab13fcf4113496e1cc308
1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 from copy
import deepcopy
19 from http
import HTTPStatus
21 from uuid
import uuid4
23 from osm_common
.dbbase
import DbBase
, DbException
24 from osm_common
.dbmongo
import deep_update
27 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
30 class DbMemory(DbBase
):
31 def __init__(self
, logger_name
="db", lock
=False):
32 super().__init
__(logger_name
, lock
)
35 def db_connect(self
, config
):
38 :param config: Configuration of database
39 :return: None or raises DbException on error
41 if "logger_name" in config
:
42 self
.logger
= logging
.getLogger(config
["logger_name"])
43 master_key
= config
.get("commonkey") or config
.get("masterpassword")
45 self
.set_secret_key(master_key
)
48 def _format_filter(q_filter
):
50 # split keys with ANYINDEX in this way:
51 # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
53 for k
, v
in q_filter
.items():
55 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
57 k
= kleft
+ ".ANYINDEX"
59 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
60 deep_update(db_filter
, {k
: db_v
})
64 def _find(self
, table
, q_filter
):
65 def recursive_find(key_list
, key_next_index
, content
, oper
, target
):
66 if key_next_index
== len(key_list
) or content
is None:
68 if oper
in ("eq", "cont"):
69 if isinstance(target
, list):
70 if isinstance(content
, list):
72 content_item
in target
for content_item
in content
74 return content
in target
75 elif isinstance(content
, list):
76 return target
in content
78 return content
== target
79 elif oper
in ("neq", "ne", "ncont"):
80 if isinstance(target
, list):
81 if isinstance(content
, list):
83 content_item
not in target
84 for content_item
in content
86 return content
not in target
87 elif isinstance(content
, list):
88 return target
not in content
90 return content
!= target
92 return content
> target
94 return content
>= target
96 return content
< target
98 return content
<= target
101 "Unknown filter operator '{}' in key '{}'".format(
102 oper
, ".".join(key_list
)
104 http_code
=HTTPStatus
.BAD_REQUEST
,
109 elif isinstance(content
, dict):
110 return recursive_find(
113 content
.get(key_list
[key_next_index
]),
117 elif isinstance(content
, list):
118 look_for_match
= True # when there is a match return immediately
119 if (target
is None) != (
120 oper
in ("neq", "ne", "ncont")
121 ): # one True and other False (Xor)
123 False # when there is not a match return immediately
126 for content_item
in content
:
127 if key_list
[key_next_index
] == "ANYINDEX" and isinstance(v
, dict):
129 for k2
, v2
in target
.items():
130 k_new_list
= k2
.split(".")
132 if k_new_list
[-1] in (
143 new_operator
= k_new_list
.pop()
144 if not recursive_find(
145 k_new_list
, 0, content_item
, new_operator
, v2
151 matches
= recursive_find(
152 key_list
, key_next_index
, content_item
, oper
, target
154 if matches
== look_for_match
:
156 if key_list
[key_next_index
].isdecimal() and int(
157 key_list
[key_next_index
]
159 matches
= recursive_find(
162 content
[int(key_list
[key_next_index
])],
166 if matches
== look_for_match
:
168 return not look_for_match
169 else: # content is not dict, nor list neither None, so not found
170 if oper
in ("neq", "ne", "ncont"):
171 return target
is not None
173 return target
is None
175 for i
, row
in enumerate(self
.db
.get(table
, ())):
176 q_filter
= q_filter
or {}
177 for k
, v
in q_filter
.items():
178 k_list
= k
.split(".")
191 operator
= k_list
.pop()
192 matches
= recursive_find(k_list
, 0, row
, operator
, v
)
199 def get_list(self
, table
, q_filter
=None):
201 Obtain a list of entries matching q_filter
202 :param table: collection or table
203 :param q_filter: Filter
204 :return: a list (can be empty) with the found entries. Raises DbException on error
209 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
210 result
.append(deepcopy(row
))
214 except Exception as e
: # TODO refine
215 raise DbException(str(e
))
217 def count(self
, table
, q_filter
=None):
219 Count the number of entries matching q_filter
220 :param table: collection or table
221 :param q_filter: Filter
222 :return: number of entries found (can be zero)
223 :raise: DbException on error
227 return sum(1 for x
in self
._find
(table
, self
._format
_filter
(q_filter
)))
230 except Exception as e
: # TODO refine
231 raise DbException(str(e
))
233 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
235 Obtain one entry matching q_filter
236 :param table: collection or table
237 :param q_filter: Filter
238 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
239 it raises a DbException
240 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
241 that it raises a DbException
242 :return: The requested element, or None
247 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
252 "Found more than one entry with filter='{}'".format(
255 HTTPStatus
.CONFLICT
.value
,
258 if not result
and fail_on_empty
:
260 "Not found entry with filter='{}'".format(q_filter
),
261 HTTPStatus
.NOT_FOUND
,
263 return deepcopy(result
)
264 except Exception as e
: # TODO refine
265 raise DbException(str(e
))
267 def del_list(self
, table
, q_filter
=None):
269 Deletes all entries that match q_filter
270 :param table: collection or table
271 :param q_filter: Filter
272 :return: Dict with the number of entries deleted
277 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
279 deleted
= len(id_list
)
280 for i
in reversed(id_list
):
281 del self
.db
[table
][i
]
282 return {"deleted": deleted
}
285 except Exception as e
: # TODO refine
286 raise DbException(str(e
))
288 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
290 Deletes one entry that matches q_filter
291 :param table: collection or table
292 :param q_filter: Filter
293 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
294 which case it raises a DbException
295 :return: Dict with the number of entries deleted
299 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
304 "Not found entry with filter='{}'".format(q_filter
),
305 HTTPStatus
.NOT_FOUND
,
308 del self
.db
[table
][i
]
309 return {"deleted": 1}
310 except Exception as e
: # TODO refine
311 raise DbException(str(e
))
324 Modifies an entry at database
325 :param db_item: entry of the table to update
326 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
327 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
328 ignored. If not exist, it is ignored
329 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
330 if exist in the array is removed. If not exist, it is ignored
331 :param pull_list: Same as pull but values are arrays where each item is removed from the array
332 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
333 is appended to the end of the array
334 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
336 :return: True if database has been changed, False if not; Exception on error
339 def _iterate_keys(k
, db_nested
, populate
=True):
340 k_list
= k
.split(".")
341 k_item_prev
= k_list
[0]
343 if k_item_prev
not in db_nested
and populate
:
345 db_nested
[k_item_prev
] = None
346 for k_item
in k_list
[1:]:
347 if isinstance(db_nested
[k_item_prev
], dict):
348 if k_item
not in db_nested
[k_item_prev
]:
351 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
354 db_nested
[k_item_prev
][k_item
] = None
355 elif isinstance(db_nested
[k_item_prev
], list) and k_item
.isdigit():
356 # extend list with Nones if index greater than list
358 if k_item
>= len(db_nested
[k_item_prev
]):
361 "Cannot set '{}', index too large '{}'".format(
366 db_nested
[k_item_prev
] += [None] * (
367 k_item
- len(db_nested
[k_item_prev
]) + 1
369 elif db_nested
[k_item_prev
] is None:
372 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
375 db_nested
[k_item_prev
] = {k_item
: None}
376 else: # number, string, boolean, ... or list but with not integer key
378 "Cannot set '{}' on existing '{}={}'".format(
379 k
, k_item_prev
, db_nested
[k_item_prev
]
382 db_nested
= db_nested
[k_item_prev
]
384 return db_nested
, k_item_prev
, populated
389 for dot_k
, v
in update_dict
.items():
390 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
)
391 dict_to_update
[key_to_update
] = v
396 dict_to_update
, key_to_update
, _
= _iterate_keys(
397 dot_k
, db_item
, populate
=False
399 del dict_to_update
[key_to_update
]
404 for dot_k
, v
in pull
.items():
406 dict_to_update
, key_to_update
, _
= _iterate_keys(
407 dot_k
, db_item
, populate
=False
411 if key_to_update
not in dict_to_update
:
413 if not isinstance(dict_to_update
[key_to_update
], list):
415 "Cannot pull '{}'. Target is not a list".format(dot_k
)
417 while v
in dict_to_update
[key_to_update
]:
418 dict_to_update
[key_to_update
].remove(v
)
421 for dot_k
, v
in pull_list
.items():
422 if not isinstance(v
, list):
424 "Invalid content at pull_list, '{}' must be an array".format(
427 http_code
=HTTPStatus
.BAD_REQUEST
,
430 dict_to_update
, key_to_update
, _
= _iterate_keys(
431 dot_k
, db_item
, populate
=False
435 if key_to_update
not in dict_to_update
:
437 if not isinstance(dict_to_update
[key_to_update
], list):
439 "Cannot pull_list '{}'. Target is not a list".format(dot_k
)
442 while single_v
in dict_to_update
[key_to_update
]:
443 dict_to_update
[key_to_update
].remove(single_v
)
446 for dot_k
, v
in push
.items():
447 dict_to_update
, key_to_update
, populated
= _iterate_keys(
451 isinstance(dict_to_update
, dict)
452 and key_to_update
not in dict_to_update
454 dict_to_update
[key_to_update
] = [v
]
456 elif populated
and dict_to_update
[key_to_update
] is None:
457 dict_to_update
[key_to_update
] = [v
]
459 elif not isinstance(dict_to_update
[key_to_update
], list):
461 "Cannot push '{}'. Target is not a list".format(dot_k
)
464 dict_to_update
[key_to_update
].append(v
)
467 for dot_k
, v
in push_list
.items():
468 if not isinstance(v
, list):
470 "Invalid content at push_list, '{}' must be an array".format(
473 http_code
=HTTPStatus
.BAD_REQUEST
,
475 dict_to_update
, key_to_update
, populated
= _iterate_keys(
479 isinstance(dict_to_update
, dict)
480 and key_to_update
not in dict_to_update
482 dict_to_update
[key_to_update
] = v
.copy()
484 elif populated
and dict_to_update
[key_to_update
] is None:
485 dict_to_update
[key_to_update
] = v
.copy()
487 elif not isinstance(dict_to_update
[key_to_update
], list):
489 "Cannot push '{}'. Target is not a list".format(dot_k
),
490 http_code
=HTTPStatus
.CONFLICT
,
493 dict_to_update
[key_to_update
] += v
499 except Exception as e
: # TODO refine
500 raise DbException(str(e
))
515 Modifies an entry at database
516 :param table: collection or table
517 :param q_filter: Filter
518 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
519 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
520 it raises a DbException
521 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
522 ignored. If not exist, it is ignored
523 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
524 if exist in the array is removed. If not exist, it is ignored
525 :param pull_list: Same as pull but values are arrays where each item is removed from the array
526 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
527 is appended to the end of the array
528 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
530 :return: Dict with the number of entries modified. None if no matching is found.
533 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
534 updated
= self
._update
(
543 return {"updated": 1 if updated
else 0}
547 "Not found entry with _id='{}'".format(q_filter
),
548 HTTPStatus
.NOT_FOUND
,
563 """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
567 for _
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
579 # if not found and fail_on_empty:
580 # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
581 return {"updated": updated
} if found
else None
583 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
585 Replace the content of an entry
586 :param table: collection or table
587 :param _id: internal database id
588 :param indata: content to replace
589 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
590 it raises a DbException
591 :return: Dict with the number of entries replaced
595 for i
, _
in self
._find
(table
, self
._format
_filter
({"_id": _id
})):
600 "Not found entry with _id='{}'".format(_id
),
601 HTTPStatus
.NOT_FOUND
,
604 self
.db
[table
][i
] = deepcopy(indata
)
605 return {"updated": 1}
608 except Exception as e
: # TODO refine
609 raise DbException(str(e
))
611 def create(self
, table
, indata
):
613 Add a new entry at database
614 :param table: collection or table
615 :param indata: content to be added
616 :return: database '_id' of the inserted element. Raises a DbException on error
619 id = indata
.get("_id")
624 if table
not in self
.db
:
626 self
.db
[table
].append(deepcopy(indata
))
628 except Exception as e
: # TODO refine
629 raise DbException(str(e
))
631 def create_list(self
, table
, indata_list
):
633 Add a new entry at database
634 :param table: collection or table
635 :param indata_list: list content to be added
636 :return: list of inserted 'id's. Raises a DbException on error
641 for indata
in indata_list
:
642 _id
= indata
.get("_id")
647 if table
not in self
.db
:
649 self
.db
[table
].append(deepcopy(indata
))
652 except Exception as e
: # TODO refine
653 raise DbException(str(e
))
656 if __name__
== "__main__":
659 db
.create("test", {"_id": 1, "data": 1})
660 db
.create("test", {"_id": 2, "data": 2})
661 db
.create("test", {"_id": 3, "data": 3})
662 print("must be 3 items:", db
.get_list("test"))
663 print("must return item 2:", db
.get_list("test", {"_id": 2}))
664 db
.del_one("test", {"_id": 2})
665 print("must be emtpy:", db
.get_list("test", {"_id": 2}))