1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
19 from osm_common
.dbbase
import DbException
, DbBase
20 from osm_common
.dbmongo
import deep_update
21 from http
import HTTPStatus
22 from uuid
import uuid4
23 from copy
import deepcopy
25 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
28 class DbMemory(DbBase
):
30 def __init__(self
, logger_name
='db', lock
=False):
31 super().__init
__(logger_name
, lock
)
34 def db_connect(self
, config
):
37 :param config: Configuration of database
38 :return: None or raises DbException on error
40 if "logger_name" in config
:
41 self
.logger
= logging
.getLogger(config
["logger_name"])
42 master_key
= config
.get("commonkey") or config
.get("masterpassword")
44 self
.set_secret_key(master_key
)
47 def _format_filter(q_filter
):
49 # split keys with ANYINDEX in this way:
50 # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v } -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
52 for k
, v
in q_filter
.items():
54 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
56 k
= kleft
+ ".ANYINDEX"
58 kleft
, _
, kright
= k
.rpartition(".ANYINDEX.")
59 deep_update(db_filter
, {k
: db_v
})
63 def _find(self
, table
, q_filter
):
65 def recursive_find(key_list
, key_next_index
, content
, oper
, target
):
66 if key_next_index
== len(key_list
) or content
is None:
68 if oper
in ("eq", "cont"):
69 if isinstance(target
, list):
70 if isinstance(content
, list):
71 return any(content_item
in target
for content_item
in content
)
72 return content
in target
73 elif isinstance(content
, list):
74 return target
in content
76 return content
== target
77 elif oper
in ("neq", "ne", "ncont"):
78 if isinstance(target
, list):
79 if isinstance(content
, list):
80 return all(content_item
not in target
for content_item
in content
)
81 return content
not in target
82 elif isinstance(content
, list):
83 return target
not in content
85 return content
!= target
87 return content
> target
89 return content
>= target
91 return content
< target
93 return content
<= target
95 raise DbException("Unknown filter operator '{}' in key '{}'".
96 format(oper
, ".".join(key_list
)), http_code
=HTTPStatus
.BAD_REQUEST
)
100 elif isinstance(content
, dict):
101 return recursive_find(key_list
, key_next_index
+ 1, content
.get(key_list
[key_next_index
]), oper
,
103 elif isinstance(content
, list):
104 look_for_match
= True # when there is a match return immediately
105 if (target
is None) != (oper
in ("neq", "ne", "ncont")): # one True and other False (Xor)
106 look_for_match
= False # when there is not a match return immediately
108 for content_item
in content
:
109 if key_list
[key_next_index
] == "ANYINDEX" and isinstance(v
, dict):
111 for k2
, v2
in target
.items():
112 k_new_list
= k2
.split(".")
114 if k_new_list
[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
115 new_operator
= k_new_list
.pop()
116 if not recursive_find(k_new_list
, 0, content_item
, new_operator
, v2
):
121 matches
= recursive_find(key_list
, key_next_index
, content_item
, oper
, target
)
122 if matches
== look_for_match
:
124 if key_list
[key_next_index
].isdecimal() and int(key_list
[key_next_index
]) < len(content
):
125 matches
= recursive_find(key_list
, key_next_index
+ 1, content
[int(key_list
[key_next_index
])],
127 if matches
== look_for_match
:
129 return not look_for_match
130 else: # content is not dict, nor list neither None, so not found
131 if oper
in ("neq", "ne", "ncont"):
132 return target
is not None
134 return target
is None
136 for i
, row
in enumerate(self
.db
.get(table
, ())):
137 q_filter
= q_filter
or {}
138 for k
, v
in q_filter
.items():
139 k_list
= k
.split(".")
141 if k_list
[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
142 operator
= k_list
.pop()
143 matches
= recursive_find(k_list
, 0, row
, operator
, v
)
150 def get_list(self
, table
, q_filter
=None):
152 Obtain a list of entries matching q_filter
153 :param table: collection or table
154 :param q_filter: Filter
155 :return: a list (can be empty) with the found entries. Raises DbException on error
160 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
161 result
.append(deepcopy(row
))
165 except Exception as e
: # TODO refine
166 raise DbException(str(e
))
168 def count(self
, table
, q_filter
=None):
170 Count the number of entries matching q_filter
171 :param table: collection or table
172 :param q_filter: Filter
173 :return: number of entries found (can be zero)
174 :raise: DbException on error
178 return sum(1 for x
in self
._find
(table
, self
._format
_filter
(q_filter
)))
181 except Exception as e
: # TODO refine
182 raise DbException(str(e
))
184 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
186 Obtain one entry matching q_filter
187 :param table: collection or table
188 :param q_filter: Filter
189 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
190 it raises a DbException
191 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
192 that it raises a DbException
193 :return: The requested element, or None
198 for _
, row
in self
._find
(table
, self
._format
_filter
(q_filter
)):
202 raise DbException("Found more than one entry with filter='{}'".format(q_filter
),
203 HTTPStatus
.CONFLICT
.value
)
205 if not result
and fail_on_empty
:
206 raise DbException("Not found entry with filter='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
207 return deepcopy(result
)
208 except Exception as e
: # TODO refine
209 raise DbException(str(e
))
211 def del_list(self
, table
, q_filter
=None):
213 Deletes all entries that match q_filter
214 :param table: collection or table
215 :param q_filter: Filter
216 :return: Dict with the number of entries deleted
221 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
223 deleted
= len(id_list
)
224 for i
in reversed(id_list
):
225 del self
.db
[table
][i
]
226 return {"deleted": deleted
}
229 except Exception as e
: # TODO refine
230 raise DbException(str(e
))
232 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
234 Deletes one entry that matches q_filter
235 :param table: collection or table
236 :param q_filter: Filter
237 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
238 which case it raises a DbException
239 :return: Dict with the number of entries deleted
243 for i
, _
in self
._find
(table
, self
._format
_filter
(q_filter
)):
247 raise DbException("Not found entry with filter='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
249 del self
.db
[table
][i
]
250 return {"deleted": 1}
251 except Exception as e
: # TODO refine
252 raise DbException(str(e
))
254 def _update(self
, db_item
, update_dict
, unset
=None, pull
=None, push
=None):
256 Modifies an entry at database
257 :param db_item: entry of the table to update
258 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
259 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
260 ignored. If not exist, it is ignored
261 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
262 if exist in the array is removed. If not exist, it is ignored
263 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
264 is appended to the end of the array
265 :return: True if database has been changed, False if not; Exception on error
267 def _iterate_keys(k
, db_nested
, populate
=True):
268 k_list
= k
.split(".")
269 k_item_prev
= k_list
[0]
271 if k_item_prev
not in db_nested
and populate
:
273 db_nested
[k_item_prev
] = None
274 for k_item
in k_list
[1:]:
275 if isinstance(db_nested
[k_item_prev
], dict):
276 if k_item
not in db_nested
[k_item_prev
]:
278 raise DbException("Cannot set '{}', not existing '{}'".format(k
, k_item
))
280 db_nested
[k_item_prev
][k_item
] = None
281 elif isinstance(db_nested
[k_item_prev
], list) and k_item
.isdigit():
282 # extend list with Nones if index greater than list
284 if k_item
>= len(db_nested
[k_item_prev
]):
286 raise DbException("Cannot set '{}', index too large '{}'".format(k
, k_item
))
288 db_nested
[k_item_prev
] += [None] * (k_item
- len(db_nested
[k_item_prev
]) + 1)
289 elif db_nested
[k_item_prev
] is None:
291 raise DbException("Cannot set '{}', not existing '{}'".format(k
, k_item
))
293 db_nested
[k_item_prev
] = {k_item
: None}
294 else: # number, string, boolean, ... or list but with not integer key
295 raise DbException("Cannot set '{}' on existing '{}={}'".format(k
, k_item_prev
,
296 db_nested
[k_item_prev
]))
297 db_nested
= db_nested
[k_item_prev
]
299 return db_nested
, k_item_prev
, populated
304 for dot_k
, v
in update_dict
.items():
305 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
)
306 dict_to_update
[key_to_update
] = v
311 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
, populate
=False)
312 del dict_to_update
[key_to_update
]
317 for dot_k
, v
in pull
.items():
319 dict_to_update
, key_to_update
, _
= _iterate_keys(dot_k
, db_item
, populate
=False)
322 if key_to_update
not in dict_to_update
:
324 if not isinstance(dict_to_update
[key_to_update
], list):
325 raise DbException("Cannot pull '{}'. Target is not a list".format(dot_k
))
326 while v
in dict_to_update
[key_to_update
]:
327 dict_to_update
[key_to_update
].remove(v
)
330 for dot_k
, v
in push
.items():
331 dict_to_update
, key_to_update
, populated
= _iterate_keys(dot_k
, db_item
)
332 if isinstance(dict_to_update
, dict) and key_to_update
not in dict_to_update
:
333 dict_to_update
[key_to_update
] = [v
]
335 elif populated
and dict_to_update
[key_to_update
] is None:
336 dict_to_update
[key_to_update
] = [v
]
338 elif not isinstance(dict_to_update
[key_to_update
], list):
339 raise DbException("Cannot push '{}'. Target is not a list".format(dot_k
))
341 dict_to_update
[key_to_update
].append(v
)
347 except Exception as e
: # TODO refine
348 raise DbException(str(e
))
350 def set_one(self
, table
, q_filter
, update_dict
, fail_on_empty
=True, unset
=None, pull
=None, push
=None):
352 Modifies an entry at database
353 :param table: collection or table
354 :param q_filter: Filter
355 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
356 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
357 it raises a DbException
358 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
359 ignored. If not exist, it is ignored
360 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
361 if exist in the array is removed. If not exist, it is ignored
362 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
363 is appended to the end of the array
364 :return: Dict with the number of entries modified. None if no matching is found.
367 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
368 updated
= self
._update
(db_item
, update_dict
, unset
=unset
, pull
=pull
, push
=push
)
369 return {"updated": 1 if updated
else 0}
372 raise DbException("Not found entry with _id='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
375 def set_list(self
, table
, q_filter
, update_dict
, fail_on_empty
=True, unset
=None, pull
=None, push
=None):
378 for i
, db_item
in self
._find
(table
, self
._format
_filter
(q_filter
)):
379 if self
._update
(db_item
, update_dict
, unset
=unset
, pull
=pull
, push
=push
):
381 if i
== 0 and fail_on_empty
:
382 raise DbException("Not found entry with _id='{}'".format(q_filter
), HTTPStatus
.NOT_FOUND
)
383 return {"updated": updated
} if i
else None
385 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
387 Replace the content of an entry
388 :param table: collection or table
389 :param _id: internal database id
390 :param indata: content to replace
391 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
392 it raises a DbException
393 :return: Dict with the number of entries replaced
397 for i
, _
in self
._find
(table
, self
._format
_filter
({"_id": _id
})):
401 raise DbException("Not found entry with _id='{}'".format(_id
), HTTPStatus
.NOT_FOUND
)
403 self
.db
[table
][i
] = deepcopy(indata
)
404 return {"updated": 1}
407 except Exception as e
: # TODO refine
408 raise DbException(str(e
))
410 def create(self
, table
, indata
):
412 Add a new entry at database
413 :param table: collection or table
414 :param indata: content to be added
415 :return: database id of the inserted element. Raises a DbException on error
418 id = indata
.get("_id")
423 if table
not in self
.db
:
425 self
.db
[table
].append(deepcopy(indata
))
427 except Exception as e
: # TODO refine
428 raise DbException(str(e
))
430 def create_list(self
, table
, indata_list
):
432 Add a new entry at database
433 :param table: collection or table
434 :param indata_list: list content to be added
435 :return: database ids of the inserted element. Raises a DbException on error
440 for indata
in indata_list
:
441 _id
= indata
.get("_id")
446 if table
not in self
.db
:
448 self
.db
[table
].append(deepcopy(indata
))
451 except Exception as e
: # TODO refine
452 raise DbException(str(e
))
455 if __name__
== '__main__':
458 db
.create("test", {"_id": 1, "data": 1})
459 db
.create("test", {"_id": 2, "data": 2})
460 db
.create("test", {"_id": 3, "data": 3})
461 print("must be 3 items:", db
.get_list("test"))
462 print("must return item 2:", db
.get_list("test", {"_id": 2}))
463 db
.del_one("test", {"_id": 2})
464 print("must be emtpy:", db
.get_list("test", {"_id": 2}))