6ad93d66f7a1fa63e7065d5664e4618e87ab349c
1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
19 from osm_common
.dbbase
import DbException
, DbBase
20 from osm_common
.dbmongo
import deep_update
21 from http
import HTTPStatus
22 from uuid
import uuid4
23 from copy
import deepcopy
25 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
28 class DbMemory(DbBase
):
30 def __init__(self
, logger_name
='db', lock
=False):
31 super().__init
__(logger_name
, lock
)
34 def db_connect(self
, config
):
37 :param config: Configuration of database
38 :return: None or raises DbException on error
40 if "logger_name" in config
:
41 self
.logger
= logging
.getLogger(config
["logger_name"])
42 master_key
= config
.get("commonkey") or config
.get("masterpassword")
44 self
.set_secret_key(master_key
)
47 def _format_filter(q_filter
):
49 # split keys with ANYINDEX in this way:
50 # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
52 for k
, v
in q_filter
.items():
54 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
56 k
= kleft
+ ".ANYINDEX"
58 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
59 deep_update(db_filter
, {k
: db_v
})
63 def _find(self
, table
, q_filter
):
65 def recursive_find(key_list
, key_next_index
, content
, oper
, target
):
66 if key_next_index
== len(key_list
) or content
is None:
68 if oper
in ("eq", "cont"):
69 if isinstance(target
, list):
70 if isinstance(content
, list):
71 return any(content_item
in target
for content_item
in content
)
72 return content
in target
73 elif isinstance(content
, list):
74 return target
in content
76 return content
== target
77 elif oper
in ("neq", "ne", "ncont"):
78 if isinstance(target
, list):
79 if isinstance(content
, list):
80 return all(content_item
not in target
for content_item
in content
)
81 return content
not in target
82 elif isinstance(content
, list):
83 return target
not in content
85 return content
!= target
87 return content
> target
89 return content
>= target
91 return content
< target
93 return content
<= target
95 raise DbException("Unknown filter operator '{}' in key '{}'".
96 format(oper
, ".".join(key_list
)), http_code
=HTTPStatus
.BAD_REQUEST
)
100 elif isinstance(content
, dict):
101 return recursive_find(key_list
, key_next_index
+ 1, content
.get(key_list
[key_next_index
]), oper
,
103 elif isinstance(content
, list):
104 look_for_match
= True # when there is a match return immediately
105 if (target
is None) != (oper
in ("neq", "ne", "ncont")): # one True and other False (Xor)
106 look_for_match
= False # when there is not a match return immediately
108 for content_item
in content
:
109 if key_list
[key_next_index
] == "ANYINDEX" and isinstance(v
, dict):
111 for k2
, v2
in target
.items():
112 k_new_list
= k2
.split(".")
114 if k_new_list
[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
115 new_operator
= k_new_list
.pop()
116 if not recursive_find(k_new_list
, 0, content_item
, new_operator
, v2
):
121 matches
= recursive_find(key_list
, key_next_index
, content_item
, oper
, target
)
122 if matches
== look_for_match
:
124 if key_list
[key_next_index
].isdecimal() and int(key_list
[key_next_index
]) < len(content
):
125 matches
= recursive_find(key_list
, key_next_index
+ 1, content
[int(key_list
[key_next_index
])],
127 if matches
== look_for_match
:
129 return not look_for_match
130 else: # content is not dict, nor list neither None, so not found
131 if oper
in ("neq", "ne", "ncont"):
132 return target
is not None
134 return target
is None
136 for i
, row
in enumerate(self
.db
.get(table
, ())):
137 q_filter
= q_filter
or {}
138 for k
, v
in q_filter
.items():
139 k_list
= k
.split(".")
141 if k_list
[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
142 operator
= k_list
.pop()
143 matches
= recursive_find(k_list
, 0, row
, operator
, v
)
150 def get_list(self
, table
, q_filter
=None):
152 Obtain a list of entries matching q_filter
153 :param table: collection or table
154 :param q_filter: Filter
155 :return: a list (can be empty) with the found entries. Raises DbException on error
160 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
161 result
.append(deepcopy(row
))
165 except Exception as e
: # TODO refine
166 raise DbException(str(e
))
168 def count(self
, table
, q_filter
=None):
170 Count the number of entries matching q_filter
171 :param table: collection or table
172 :param q_filter: Filter
173 :return: number of entries found (can be zero)
174 :raise: DbException on error
178 return sum(1 for x
in self
._find
(table
, self
._format
_filter
(q_filter
)))
181 except Exception as e
: # TODO refine
182 raise DbException(str(e
))
184 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
186 Obtain one entry matching q_filter
187 :param table: collection or table
188 :param q_filter: Filter
189 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
190 it raises a DbException
191 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
192 that it raises a DbException
193 :return: The requested element, or None
198 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
202 raise DbException("Found more than one entry with filter='{}'".format(q_filter
),
203 HTTPStatus
.CONFLICT
.value
)
205 if not result
and fail_on_empty
:
206 raise DbException("Not found entry with filter='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
207 return deepcopy(result
)
208 except Exception as e
: # TODO refine
209 raise DbException(str(e
))
211 def del_list(self
, table
, q_filter
=None):
213 Deletes all entries that match q_filter
214 :param table: collection or table
215 :param q_filter: Filter
216 :return: Dict with the number of entries deleted
221 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
223 deleted
= len(id_list
)
224 for i
in reversed(id_list
):
225 del self
.db
[table
][i
]
226 return {"deleted": deleted
}
229 except Exception as e
: # TODO refine
230 raise DbException(str(e
))
232 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
234 Deletes one entry that matches q_filter
235 :param table: collection or table
236 :param q_filter: Filter
237 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
238 which case it raises a DbException
239 :return: Dict with the number of entries deleted
243 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
247 raise DbException("Not found entry with filter='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
249 del self
.db
[table
][i
]
250 return {"deleted": 1}
251 except Exception as e
: # TODO refine
252 raise DbException(str(e
))
254 def _update(self
, db_item
, update_dict
, unset
=None, pull
=None, push
=None, push_list
=None):
256 Modifies an entry at database
257 :param db_item: entry of the table to update
258 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
259 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
260 ignored. If not exist, it is ignored
261 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
262 if exist in the array is removed. If not exist, it is ignored
263 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
264 is appended to the end of the array
265 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
267 :return: True if database has been changed, False if not; Exception on error
269 def _iterate_keys(k
, db_nested
, populate
=True):
270 k_list
= k
.split(".")
271 k_item_prev
= k_list
[0]
273 if k_item_prev
not in db_nested
and populate
:
275 db_nested
[k_item_prev
] = None
276 for k_item
in k_list
[1:]:
277 if isinstance(db_nested
[k_item_prev
], dict):
278 if k_item
not in db_nested
[k_item_prev
]:
280 raise DbException("Cannot set '{}', not existing '{}'".format(k
, k_item
))
282 db_nested
[k_item_prev
][k_item
] = None
283 elif isinstance(db_nested
[k_item_prev
], list) and k_item
.isdigit():
284 # extend list with Nones if index greater than list
286 if k_item
>= len(db_nested
[k_item_prev
]):
288 raise DbException("Cannot set '{}', index too large '{}'".format(k
, k_item
))
290 db_nested
[k_item_prev
] += [None] * (k_item
- len(db_nested
[k_item_prev
]) + 1)
291 elif db_nested
[k_item_prev
] is None:
293 raise DbException("Cannot set '{}', not existing '{}'".format(k
, k_item
))
295 db_nested
[k_item_prev
] = {k_item
: None}
296 else: # number, string, boolean, ... or list but with not integer key
297 raise DbException("Cannot set '{}' on existing '{}={}'".format(k
, k_item_prev
,
298 db_nested
[k_item_prev
]))
299 db_nested
= db_nested
[k_item_prev
]
301 return db_nested
, k_item_prev
, populated
306 for dot_k
, v
in update_dict
.items():
307 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
)
308 dict_to_update
[key_to_update
] = v
313 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
, populate
=False)
314 del dict_to_update
[key_to_update
]
319 for dot_k
, v
in pull
.items():
321 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
, populate
=False)
324 if key_to_update
not in dict_to_update
:
326 if not isinstance(dict_to_update
[key_to_update
], list):
327 raise DbException("Cannot pull '{}'. Target is not a list".format(dot_k
))
328 while v
in dict_to_update
[key_to_update
]:
329 dict_to_update
[key_to_update
].remove(v
)
332 for dot_k
, v
in push
.items():
333 dict_to_update
, key_to_update
, populated
= _iterate_keys(dot_k
, db_item
)
334 if isinstance(dict_to_update
, dict) and key_to_update
not in dict_to_update
:
335 dict_to_update
[key_to_update
] = [v
]
337 elif populated
and dict_to_update
[key_to_update
] is None:
338 dict_to_update
[key_to_update
] = [v
]
340 elif not isinstance(dict_to_update
[key_to_update
], list):
341 raise DbException("Cannot push '{}'. Target is not a list".format(dot_k
))
343 dict_to_update
[key_to_update
].append(v
)
346 for dot_k
, v
in push_list
.items():
347 if not isinstance(v
, list):
348 raise DbException("Invalid content at push_list, '{}' must be an array".format(dot_k
),
349 http_code
=HTTPStatus
.BAD_REQUEST
)
350 dict_to_update
, key_to_update
, populated
= _iterate_keys(dot_k
, db_item
)
351 if isinstance(dict_to_update
, dict) and key_to_update
not in dict_to_update
:
352 dict_to_update
[key_to_update
] = v
.copy()
354 elif populated
and dict_to_update
[key_to_update
] is None:
355 dict_to_update
[key_to_update
] = v
.copy()
357 elif not isinstance(dict_to_update
[key_to_update
], list):
358 raise DbException("Cannot push '{}'. Target is not a list".format(dot_k
),
359 http_code
=HTTPStatus
.CONFLICT
)
361 dict_to_update
[key_to_update
] += v
367 except Exception as e
: # TODO refine
368 raise DbException(str(e
))
370 def set_one(self
, table
, q_filter
, update_dict
, fail_on_empty
=True, unset
=None, pull
=None, push
=None,
373 Modifies an entry at database
374 :param table: collection or table
375 :param q_filter: Filter
376 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
377 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
378 it raises a DbException
379 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
380 ignored. If not exist, it is ignored
381 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
382 if exist in the array is removed. If not exist, it is ignored
383 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
384 is appended to the end of the array
385 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
387 :return: Dict with the number of entries modified. None if no matching is found.
390 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
391 updated
= self
._update
(db_item
, update_dict
, unset
=unset
, pull
=pull
, push
=push
, push_list
=push_list
)
392 return {"updated": 1 if updated
else 0}
395 raise DbException("Not found entry with _id='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
398 def set_list(self
, table
, q_filter
, update_dict
, unset
=None, pull
=None, push
=None, push_list
=None):
399 """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
403 for _
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
405 if self
._update
(db_item
, update_dict
, unset
=unset
, pull
=pull
, push
=push
, push_list
=push_list
):
407 # if not found and fail_on_empty:
408 # raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
409 return {"updated": updated
} if found
else None
411 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
413 Replace the content of an entry
414 :param table: collection or table
415 :param _id: internal database id
416 :param indata: content to replace
417 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
418 it raises a DbException
419 :return: Dict with the number of entries replaced
423 for i
, _
in self
._find
(table
, self
._format
_filter
({"_id": _id
})):
427 raise DbException("Not found entry with _id='{}'".format(_id
), HTTPStatus
.NOT_FOUND
)
429 self
.db
[table
][i
] = deepcopy(indata
)
430 return {"updated": 1}
433 except Exception as e
: # TODO refine
434 raise DbException(str(e
))
436 def create(self
, table
, indata
):
438 Add a new entry at database
439 :param table: collection or table
440 :param indata: content to be added
441 :return: database '_id' of the inserted element. Raises a DbException on error
444 id = indata
.get("_id")
449 if table
not in self
.db
:
451 self
.db
[table
].append(deepcopy(indata
))
453 except Exception as e
: # TODO refine
454 raise DbException(str(e
))
456 def create_list(self
, table
, indata_list
):
458 Add a new entry at database
459 :param table: collection or table
460 :param indata_list: list content to be added
461 :return: list of inserted 'id's. Raises a DbException on error
466 for indata
in indata_list
:
467 _id
= indata
.get("_id")
472 if table
not in self
.db
:
474 self
.db
[table
].append(deepcopy(indata
))
477 except Exception as e
: # TODO refine
478 raise DbException(str(e
))
481 if __name__
== '__main__':
484 db
.create("test", {"_id": 1, "data": 1})
485 db
.create("test", {"_id": 2, "data": 2})
486 db
.create("test", {"_id": 3, "data": 3})
487 print("must be 3 items:", db
.get_list("test"))
488 print("must return item 2:", db
.get_list("test", {"_id": 2}))
489 db
.del_one("test", {"_id": 2})
490 print("must be emtpy:", db
.get_list("test", {"_id": 2}))