1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
19 from osm_common
.dbbase
import DbException
, DbBase
20 from osm_common
.dbmongo
import deep_update
21 from http
import HTTPStatus
22 from uuid
import uuid4
23 from copy
import deepcopy
25 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
28 class DbMemory(DbBase
):
29 def __init__(self
, logger_name
="db", lock
=False):
30 super().__init
__(logger_name
, lock
)
33 def db_connect(self
, config
):
36 :param config: Configuration of database
37 :return: None or raises DbException on error
39 if "logger_name" in config
:
40 self
.logger
= logging
.getLogger(config
["logger_name"])
41 master_key
= config
.get("commonkey") or config
.get("masterpassword")
43 self
.set_secret_key(master_key
)
46 def _format_filter(q_filter
):
48 # split keys with ANYINDEX in this way:
49 # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
51 for k
, v
in q_filter
.items():
53 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
55 k
= kleft
+ ".ANYINDEX"
57 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
58 deep_update(db_filter
, {k
: db_v
})
62 def _find(self
, table
, q_filter
):
63 def recursive_find(key_list
, key_next_index
, content
, oper
, target
):
64 if key_next_index
== len(key_list
) or content
is None:
66 if oper
in ("eq", "cont"):
67 if isinstance(target
, list):
68 if isinstance(content
, list):
70 content_item
in target
for content_item
in content
72 return content
in target
73 elif isinstance(content
, list):
74 return target
in content
76 return content
== target
77 elif oper
in ("neq", "ne", "ncont"):
78 if isinstance(target
, list):
79 if isinstance(content
, list):
81 content_item
not in target
82 for content_item
in content
84 return content
not in target
85 elif isinstance(content
, list):
86 return target
not in content
88 return content
!= target
90 return content
> target
92 return content
>= target
94 return content
< target
96 return content
<= target
99 "Unknown filter operator '{}' in key '{}'".format(
100 oper
, ".".join(key_list
)
102 http_code
=HTTPStatus
.BAD_REQUEST
,
107 elif isinstance(content
, dict):
108 return recursive_find(
111 content
.get(key_list
[key_next_index
]),
115 elif isinstance(content
, list):
116 look_for_match
= True # when there is a match return immediately
117 if (target
is None) != (
118 oper
in ("neq", "ne", "ncont")
119 ): # one True and other False (Xor)
121 False # when there is not a match return immediately
124 for content_item
in content
:
125 if key_list
[key_next_index
] == "ANYINDEX" and isinstance(v
, dict):
127 for k2
, v2
in target
.items():
128 k_new_list
= k2
.split(".")
130 if k_new_list
[-1] in (
141 new_operator
= k_new_list
.pop()
142 if not recursive_find(
143 k_new_list
, 0, content_item
, new_operator
, v2
149 matches
= recursive_find(
150 key_list
, key_next_index
, content_item
, oper
, target
152 if matches
== look_for_match
:
154 if key_list
[key_next_index
].isdecimal() and int(
155 key_list
[key_next_index
]
157 matches
= recursive_find(
160 content
[int(key_list
[key_next_index
])],
164 if matches
== look_for_match
:
166 return not look_for_match
167 else: # content is not dict, nor list neither None, so not found
168 if oper
in ("neq", "ne", "ncont"):
169 return target
is not None
171 return target
is None
173 for i
, row
in enumerate(self
.db
.get(table
, ())):
174 q_filter
= q_filter
or {}
175 for k
, v
in q_filter
.items():
176 k_list
= k
.split(".")
189 operator
= k_list
.pop()
190 matches
= recursive_find(k_list
, 0, row
, operator
, v
)
197 def get_list(self
, table
, q_filter
=None):
199 Obtain a list of entries matching q_filter
200 :param table: collection or table
201 :param q_filter: Filter
202 :return: a list (can be empty) with the found entries. Raises DbException on error
207 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
208 result
.append(deepcopy(row
))
212 except Exception as e
: # TODO refine
213 raise DbException(str(e
))
215 def count(self
, table
, q_filter
=None):
217 Count the number of entries matching q_filter
218 :param table: collection or table
219 :param q_filter: Filter
220 :return: number of entries found (can be zero)
221 :raise: DbException on error
225 return sum(1 for x
in self
._find
(table
, self
._format
_filter
(q_filter
)))
228 except Exception as e
: # TODO refine
229 raise DbException(str(e
))
231 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
233 Obtain one entry matching q_filter
234 :param table: collection or table
235 :param q_filter: Filter
236 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
237 it raises a DbException
238 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
239 that it raises a DbException
240 :return: The requested element, or None
245 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
250 "Found more than one entry with filter='{}'".format(
253 HTTPStatus
.CONFLICT
.value
,
256 if not result
and fail_on_empty
:
258 "Not found entry with filter='{}'".format(q_filter
),
259 HTTPStatus
.NOT_FOUND
,
261 return deepcopy(result
)
262 except Exception as e
: # TODO refine
263 raise DbException(str(e
))
265 def del_list(self
, table
, q_filter
=None):
267 Deletes all entries that match q_filter
268 :param table: collection or table
269 :param q_filter: Filter
270 :return: Dict with the number of entries deleted
275 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
277 deleted
= len(id_list
)
278 for i
in reversed(id_list
):
279 del self
.db
[table
][i
]
280 return {"deleted": deleted
}
283 except Exception as e
: # TODO refine
284 raise DbException(str(e
))
286 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
288 Deletes one entry that matches q_filter
289 :param table: collection or table
290 :param q_filter: Filter
291 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
292 which case it raises a DbException
293 :return: Dict with the number of entries deleted
297 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
302 "Not found entry with filter='{}'".format(q_filter
),
303 HTTPStatus
.NOT_FOUND
,
306 del self
.db
[table
][i
]
307 return {"deleted": 1}
308 except Exception as e
: # TODO refine
309 raise DbException(str(e
))
322 Modifies an entry at database
323 :param db_item: entry of the table to update
324 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
325 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
326 ignored. If not exist, it is ignored
327 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
328 if exist in the array is removed. If not exist, it is ignored
329 :param pull_list: Same as pull but values are arrays where each item is removed from the array
330 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
331 is appended to the end of the array
332 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
334 :return: True if database has been changed, False if not; Exception on error
337 def _iterate_keys(k
, db_nested
, populate
=True):
338 k_list
= k
.split(".")
339 k_item_prev
= k_list
[0]
341 if k_item_prev
not in db_nested
and populate
:
343 db_nested
[k_item_prev
] = None
344 for k_item
in k_list
[1:]:
345 if isinstance(db_nested
[k_item_prev
], dict):
346 if k_item
not in db_nested
[k_item_prev
]:
349 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
352 db_nested
[k_item_prev
][k_item
] = None
353 elif isinstance(db_nested
[k_item_prev
], list) and k_item
.isdigit():
354 # extend list with Nones if index greater than list
356 if k_item
>= len(db_nested
[k_item_prev
]):
359 "Cannot set '{}', index too large '{}'".format(
364 db_nested
[k_item_prev
] += [None] * (
365 k_item
- len(db_nested
[k_item_prev
]) + 1
367 elif db_nested
[k_item_prev
] is None:
370 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
373 db_nested
[k_item_prev
] = {k_item
: None}
374 else: # number, string, boolean, ... or list but with not integer key
376 "Cannot set '{}' on existing '{}={}'".format(
377 k
, k_item_prev
, db_nested
[k_item_prev
]
380 db_nested
= db_nested
[k_item_prev
]
382 return db_nested
, k_item_prev
, populated
387 for dot_k
, v
in update_dict
.items():
388 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
)
389 dict_to_update
[key_to_update
] = v
394 dict_to_update
, key_to_update
, _
= _iterate_keys(
395 dot_k
, db_item
, populate
=False
397 del dict_to_update
[key_to_update
]
402 for dot_k
, v
in pull
.items():
404 dict_to_update
, key_to_update
, _
= _iterate_keys(
405 dot_k
, db_item
, populate
=False
409 if key_to_update
not in dict_to_update
:
411 if not isinstance(dict_to_update
[key_to_update
], list):
413 "Cannot pull '{}'. Target is not a list".format(dot_k
)
415 while v
in dict_to_update
[key_to_update
]:
416 dict_to_update
[key_to_update
].remove(v
)
419 for dot_k
, v
in pull_list
.items():
420 if not isinstance(v
, list):
422 "Invalid content at pull_list, '{}' must be an array".format(
425 http_code
=HTTPStatus
.BAD_REQUEST
,
428 dict_to_update
, key_to_update
, _
= _iterate_keys(
429 dot_k
, db_item
, populate
=False
433 if key_to_update
not in dict_to_update
:
435 if not isinstance(dict_to_update
[key_to_update
], list):
437 "Cannot pull_list '{}'. Target is not a list".format(dot_k
)
440 while single_v
in dict_to_update
[key_to_update
]:
441 dict_to_update
[key_to_update
].remove(single_v
)
444 for dot_k
, v
in push
.items():
445 dict_to_update
, key_to_update
, populated
= _iterate_keys(
449 isinstance(dict_to_update
, dict)
450 and key_to_update
not in dict_to_update
452 dict_to_update
[key_to_update
] = [v
]
454 elif populated
and dict_to_update
[key_to_update
] is None:
455 dict_to_update
[key_to_update
] = [v
]
457 elif not isinstance(dict_to_update
[key_to_update
], list):
459 "Cannot push '{}'. Target is not a list".format(dot_k
)
462 dict_to_update
[key_to_update
].append(v
)
465 for dot_k
, v
in push_list
.items():
466 if not isinstance(v
, list):
468 "Invalid content at push_list, '{}' must be an array".format(
471 http_code
=HTTPStatus
.BAD_REQUEST
,
473 dict_to_update
, key_to_update
, populated
= _iterate_keys(
477 isinstance(dict_to_update
, dict)
478 and key_to_update
not in dict_to_update
480 dict_to_update
[key_to_update
] = v
.copy()
482 elif populated
and dict_to_update
[key_to_update
] is None:
483 dict_to_update
[key_to_update
] = v
.copy()
485 elif not isinstance(dict_to_update
[key_to_update
], list):
487 "Cannot push '{}'. Target is not a list".format(dot_k
),
488 http_code
=HTTPStatus
.CONFLICT
,
491 dict_to_update
[key_to_update
] += v
497 except Exception as e
: # TODO refine
498 raise DbException(str(e
))
513 Modifies an entry at database
514 :param table: collection or table
515 :param q_filter: Filter
516 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
517 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
518 it raises a DbException
519 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
520 ignored. If not exist, it is ignored
521 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
522 if exist in the array is removed. If not exist, it is ignored
523 :param pull_list: Same as pull but values are arrays where each item is removed from the array
524 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
525 is appended to the end of the array
526 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
528 :return: Dict with the number of entries modified. None if no matching is found.
531 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
532 updated
= self
._update
(
541 return {"updated": 1 if updated
else 0}
545 "Not found entry with _id='{}'".format(q_filter
),
546 HTTPStatus
.NOT_FOUND
,
561 """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
565 for _
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
577 # if not found and fail_on_empty:
578 # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
579 return {"updated": updated
} if found
else None
581 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
583 Replace the content of an entry
584 :param table: collection or table
585 :param _id: internal database id
586 :param indata: content to replace
587 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
588 it raises a DbException
589 :return: Dict with the number of entries replaced
593 for i
, _
in self
._find
(table
, self
._format
_filter
({"_id": _id
})):
598 "Not found entry with _id='{}'".format(_id
),
599 HTTPStatus
.NOT_FOUND
,
602 self
.db
[table
][i
] = deepcopy(indata
)
603 return {"updated": 1}
606 except Exception as e
: # TODO refine
607 raise DbException(str(e
))
609 def create(self
, table
, indata
):
611 Add a new entry at database
612 :param table: collection or table
613 :param indata: content to be added
614 :return: database '_id' of the inserted element. Raises a DbException on error
617 id = indata
.get("_id")
622 if table
not in self
.db
:
624 self
.db
[table
].append(deepcopy(indata
))
626 except Exception as e
: # TODO refine
627 raise DbException(str(e
))
629 def create_list(self
, table
, indata_list
):
631 Add a new entry at database
632 :param table: collection or table
633 :param indata_list: list content to be added
634 :return: list of inserted 'id's. Raises a DbException on error
639 for indata
in indata_list
:
640 _id
= indata
.get("_id")
645 if table
not in self
.db
:
647 self
.db
[table
].append(deepcopy(indata
))
650 except Exception as e
: # TODO refine
651 raise DbException(str(e
))
654 if __name__
== "__main__":
657 db
.create("test", {"_id": 1, "data": 1})
658 db
.create("test", {"_id": 2, "data": 2})
659 db
.create("test", {"_id": 3, "data": 3})
660 print("must be 3 items:", db
.get_list("test"))
661 print("must return item 2:", db
.get_list("test", {"_id": 2}))
662 db
.del_one("test", {"_id": 2})
663 print("must be emtpy:", db
.get_list("test", {"_id": 2}))