1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 from copy
import deepcopy
19 from http
import HTTPStatus
21 from uuid
import uuid4
23 from osm_common
.dbbase
import DbBase
, DbException
24 from osm_common
.dbmongo
import deep_update
27 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
30 class DbMemory(DbBase
):
31 def __init__(self
, logger_name
="db", lock
=False):
32 super().__init
__(logger_name
=logger_name
, lock
=lock
)
35 def db_connect(self
, config
):
38 :param config: Configuration of database
39 :return: None or raises DbException on error
41 if "logger_name" in config
:
42 self
.logger
= logging
.getLogger(config
["logger_name"])
43 master_key
= config
.get("commonkey") or config
.get("masterpassword")
45 self
.set_secret_key(master_key
)
48 def _format_filter(q_filter
):
50 # split keys with ANYINDEX in this way:
51 # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
53 for k
, v
in q_filter
.items():
55 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
57 k
= kleft
+ ".ANYINDEX"
59 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
60 deep_update(db_filter
, {k
: db_v
})
64 def _find(self
, table
, q_filter
):
65 def recursive_find(key_list
, key_next_index
, content
, oper
, target
):
66 if key_next_index
== len(key_list
) or content
is None:
68 if oper
in ("eq", "cont"):
69 if isinstance(target
, list):
70 if isinstance(content
, list):
72 content_item
in target
for content_item
in content
74 return content
in target
75 elif isinstance(content
, list):
76 return target
in content
78 return content
== target
79 elif oper
in ("neq", "ne", "ncont"):
80 if isinstance(target
, list):
81 if isinstance(content
, list):
83 content_item
not in target
84 for content_item
in content
86 return content
not in target
87 elif isinstance(content
, list):
88 return target
not in content
90 return content
!= target
92 return content
> target
94 return content
>= target
96 return content
< target
98 return content
<= target
101 "Unknown filter operator '{}' in key '{}'".format(
102 oper
, ".".join(key_list
)
104 http_code
=HTTPStatus
.BAD_REQUEST
,
109 elif isinstance(content
, dict):
110 return recursive_find(
113 content
.get(key_list
[key_next_index
]),
117 elif isinstance(content
, list):
118 look_for_match
= True # when there is a match return immediately
119 if (target
is None) != (
120 oper
in ("neq", "ne", "ncont")
121 ): # one True and other False (Xor)
123 False # when there is not a match return immediately
126 for content_item
in content
:
127 if key_list
[key_next_index
] == "ANYINDEX" and isinstance(v
, dict):
130 for k2
, v2
in target
.items():
131 k_new_list
= k2
.split(".")
133 if k_new_list
[-1] in (
144 new_operator
= k_new_list
.pop()
145 if not recursive_find(
146 k_new_list
, 0, content_item
, new_operator
, v2
152 matches
= recursive_find(
153 key_list
, key_next_index
, content_item
, oper
, target
155 if matches
== look_for_match
:
157 if key_list
[key_next_index
].isdecimal() and int(
158 key_list
[key_next_index
]
160 matches
= recursive_find(
163 content
[int(key_list
[key_next_index
])],
167 if matches
== look_for_match
:
169 return not look_for_match
170 else: # content is not dict, nor list neither None, so not found
171 if oper
in ("neq", "ne", "ncont"):
172 return target
is not None
174 return target
is None
176 for i
, row
in enumerate(self
.db
.get(table
, ())):
177 q_filter
= q_filter
or {}
178 for k
, v
in q_filter
.items():
179 k_list
= k
.split(".")
192 operator
= k_list
.pop()
193 matches
= recursive_find(k_list
, 0, row
, operator
, v
)
200 def get_list(self
, table
, q_filter
=None):
202 Obtain a list of entries matching q_filter
203 :param table: collection or table
204 :param q_filter: Filter
205 :return: a list (can be empty) with the found entries. Raises DbException on error
210 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
211 result
.append(deepcopy(row
))
215 except Exception as e
: # TODO refine
216 raise DbException(str(e
))
218 def count(self
, table
, q_filter
=None):
220 Count the number of entries matching q_filter
221 :param table: collection or table
222 :param q_filter: Filter
223 :return: number of entries found (can be zero)
224 :raise: DbException on error
228 return sum(1 for x
in self
._find
(table
, self
._format
_filter
(q_filter
)))
231 except Exception as e
: # TODO refine
232 raise DbException(str(e
))
234 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
236 Obtain one entry matching q_filter
237 :param table: collection or table
238 :param q_filter: Filter
239 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
240 it raises a DbException
241 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
242 that it raises a DbException
243 :return: The requested element, or None
248 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
253 "Found more than one entry with filter='{}'".format(
256 HTTPStatus
.CONFLICT
.value
,
259 if not result
and fail_on_empty
:
261 "Not found entry with filter='{}'".format(q_filter
),
262 HTTPStatus
.NOT_FOUND
,
264 return deepcopy(result
)
265 except Exception as e
: # TODO refine
266 raise DbException(str(e
))
268 def del_list(self
, table
, q_filter
=None):
270 Deletes all entries that match q_filter
271 :param table: collection or table
272 :param q_filter: Filter
273 :return: Dict with the number of entries deleted
278 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
280 deleted
= len(id_list
)
281 for i
in reversed(id_list
):
282 del self
.db
[table
][i
]
283 return {"deleted": deleted
}
286 except Exception as e
: # TODO refine
287 raise DbException(str(e
))
289 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
291 Deletes one entry that matches q_filter
292 :param table: collection or table
293 :param q_filter: Filter
294 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
295 which case it raises a DbException
296 :return: Dict with the number of entries deleted
300 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
305 "Not found entry with filter='{}'".format(q_filter
),
306 HTTPStatus
.NOT_FOUND
,
309 del self
.db
[table
][i
]
310 return {"deleted": 1}
311 except Exception as e
: # TODO refine
312 raise DbException(str(e
))
325 Modifies an entry at database
326 :param db_item: entry of the table to update
327 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
328 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
329 ignored. If not exist, it is ignored
330 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
331 if exist in the array is removed. If not exist, it is ignored
332 :param pull_list: Same as pull but values are arrays where each item is removed from the array
333 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
334 is appended to the end of the array
335 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
337 :return: True if database has been changed, False if not; Exception on error
340 def _iterate_keys(k
, db_nested
, populate
=True):
341 k_list
= k
.split(".")
342 k_item_prev
= k_list
[0]
344 if k_item_prev
not in db_nested
and populate
:
346 db_nested
[k_item_prev
] = None
347 for k_item
in k_list
[1:]:
348 if isinstance(db_nested
[k_item_prev
], dict):
349 if k_item
not in db_nested
[k_item_prev
]:
352 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
355 db_nested
[k_item_prev
][k_item
] = None
356 elif isinstance(db_nested
[k_item_prev
], list) and k_item
.isdigit():
357 # extend list with Nones if index greater than list
359 if k_item
>= len(db_nested
[k_item_prev
]):
362 "Cannot set '{}', index too large '{}'".format(
367 db_nested
[k_item_prev
] += [None] * (
368 k_item
- len(db_nested
[k_item_prev
]) + 1
370 elif db_nested
[k_item_prev
] is None:
373 "Cannot set '{}', not existing '{}'".format(k
, k_item
)
376 db_nested
[k_item_prev
] = {k_item
: None}
377 else: # number, string, boolean, ... or list but with not integer key
379 "Cannot set '{}' on existing '{}={}'".format(
380 k
, k_item_prev
, db_nested
[k_item_prev
]
383 db_nested
= db_nested
[k_item_prev
]
385 return db_nested
, k_item_prev
, populated
390 for dot_k
, v
in update_dict
.items():
391 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
)
392 dict_to_update
[key_to_update
] = v
397 dict_to_update
, key_to_update
, _
= _iterate_keys(
398 dot_k
, db_item
, populate
=False
400 del dict_to_update
[key_to_update
]
402 except Exception as unset_error
:
403 self
.logger
.error(f
"{unset_error} occured while updating DB.")
405 for dot_k
, v
in pull
.items():
407 dict_to_update
, key_to_update
, _
= _iterate_keys(
408 dot_k
, db_item
, populate
=False
410 except Exception as pull_error
:
411 self
.logger
.error(f
"{pull_error} occured while updating DB.")
414 if key_to_update
not in dict_to_update
:
416 if not isinstance(dict_to_update
[key_to_update
], list):
418 "Cannot pull '{}'. Target is not a list".format(dot_k
)
420 while v
in dict_to_update
[key_to_update
]:
421 dict_to_update
[key_to_update
].remove(v
)
424 for dot_k
, v
in pull_list
.items():
425 if not isinstance(v
, list):
427 "Invalid content at pull_list, '{}' must be an array".format(
430 http_code
=HTTPStatus
.BAD_REQUEST
,
433 dict_to_update
, key_to_update
, _
= _iterate_keys(
434 dot_k
, db_item
, populate
=False
436 except Exception as iterate_error
:
438 f
"{iterate_error} occured while iterating keys in db update."
442 if key_to_update
not in dict_to_update
:
444 if not isinstance(dict_to_update
[key_to_update
], list):
446 "Cannot pull_list '{}'. Target is not a list".format(dot_k
)
449 while single_v
in dict_to_update
[key_to_update
]:
450 dict_to_update
[key_to_update
].remove(single_v
)
453 for dot_k
, v
in push
.items():
454 dict_to_update
, key_to_update
, populated
= _iterate_keys(
458 isinstance(dict_to_update
, dict)
459 and key_to_update
not in dict_to_update
461 dict_to_update
[key_to_update
] = [v
]
463 elif populated
and dict_to_update
[key_to_update
] is None:
464 dict_to_update
[key_to_update
] = [v
]
466 elif not isinstance(dict_to_update
[key_to_update
], list):
468 "Cannot push '{}'. Target is not a list".format(dot_k
)
471 dict_to_update
[key_to_update
].append(v
)
474 for dot_k
, v
in push_list
.items():
475 if not isinstance(v
, list):
477 "Invalid content at push_list, '{}' must be an array".format(
480 http_code
=HTTPStatus
.BAD_REQUEST
,
482 dict_to_update
, key_to_update
, populated
= _iterate_keys(
486 isinstance(dict_to_update
, dict)
487 and key_to_update
not in dict_to_update
489 dict_to_update
[key_to_update
] = v
.copy()
491 elif populated
and dict_to_update
[key_to_update
] is None:
492 dict_to_update
[key_to_update
] = v
.copy()
494 elif not isinstance(dict_to_update
[key_to_update
], list):
496 "Cannot push '{}'. Target is not a list".format(dot_k
),
497 http_code
=HTTPStatus
.CONFLICT
,
500 dict_to_update
[key_to_update
] += v
506 except Exception as e
: # TODO refine
507 raise DbException(str(e
))
522 Modifies an entry at database
523 :param table: collection or table
524 :param q_filter: Filter
525 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
526 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
527 it raises a DbException
528 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
529 ignored. If not exist, it is ignored
530 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
531 if exist in the array is removed. If not exist, it is ignored
532 :param pull_list: Same as pull but values are arrays where each item is removed from the array
533 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
534 is appended to the end of the array
535 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
537 :return: Dict with the number of entries modified. None if no matching is found.
540 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
541 updated
= self
._update
(
550 return {"updated": 1 if updated
else 0}
554 "Not found entry with _id='{}'".format(q_filter
),
555 HTTPStatus
.NOT_FOUND
,
570 """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
574 for _
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
586 # if not found and fail_on_empty:
587 # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
588 return {"updated": updated
} if found
else None
590 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
592 Replace the content of an entry
593 :param table: collection or table
594 :param _id: internal database id
595 :param indata: content to replace
596 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
597 it raises a DbException
598 :return: Dict with the number of entries replaced
602 for i
, _
in self
._find
(table
, self
._format
_filter
({"_id": _id
})):
607 "Not found entry with _id='{}'".format(_id
),
608 HTTPStatus
.NOT_FOUND
,
611 self
.db
[table
][i
] = deepcopy(indata
)
612 return {"updated": 1}
615 except Exception as e
: # TODO refine
616 raise DbException(str(e
))
618 def create(self
, table
, indata
):
620 Add a new entry at database
621 :param table: collection or table
622 :param indata: content to be added
623 :return: database '_id' of the inserted element. Raises a DbException on error
626 id = indata
.get("_id")
631 if table
not in self
.db
:
633 self
.db
[table
].append(deepcopy(indata
))
635 except Exception as e
: # TODO refine
636 raise DbException(str(e
))
638 def create_list(self
, table
, indata_list
):
640 Add a new entry at database
641 :param table: collection or table
642 :param indata_list: list content to be added
643 :return: list of inserted 'id's. Raises a DbException on error
648 for indata
in indata_list
:
649 _id
= indata
.get("_id")
654 if table
not in self
.db
:
656 self
.db
[table
].append(deepcopy(indata
))
659 except Exception as e
: # TODO refine
660 raise DbException(str(e
))
663 if __name__
== "__main__":
666 db
.create("test", {"_id": 1, "data": 1})
667 db
.create("test", {"_id": 2, "data": 2})
668 db
.create("test", {"_id": 3, "data": 3})
669 print("must be 3 items:", db
.get_list("test"))
670 print("must return item 2:", db
.get_list("test", {"_id": 2}))
671 db
.del_one("test", {"_id": 2})
672 print("must be emtpy:", db
.get_list("test", {"_id": 2}))