1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
21 from http
import HTTPStatus
22 from copy
import deepcopy
23 from Crypto
.Cipher
import AES
24 from base64
import b64decode
, b64encode
25 from osm_common
.common_utils
import FakeLock
26 from threading
import Lock
28 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
31 class DbException(Exception):
33 def __init__(self
, message
, http_code
=HTTPStatus
.NOT_FOUND
):
34 self
.http_code
= http_code
35 Exception.__init
__(self
, "database exception " + str(message
))
40 def __init__(self
, logger_name
='db', lock
=False):
43 :param logger_name: logging name
44 :param lock: Used to protect simultaneous access to the same instance class by several threads:
45 False, None: Do not protect, this object will only be accessed by one thread
46 True: This object needs to be protected by several threads accessing.
47 Lock object. Use thi Lock for the threads access protection
49 self
.logger
= logging
.getLogger(logger_name
)
50 self
.secret_key
= None # 32 bytes length array used for encrypt/decrypt
52 self
.lock
= FakeLock()
55 elif isinstance(lock
, Lock
):
58 raise ValueError("lock parameter must be a Lock classclass or boolean")
60 def db_connect(self
, config
, target_version
=None):
63 :param config: Configuration of database. Contains among others:
64 host: database host (mandatory)
65 port: database port (mandatory)
66 name: database name (mandatory)
67 user: database username
68 password: database password
69 commonkey: common OSM key used for sensible information encryption
70 materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
71 :param target_version: if provided it checks if database contains required version, raising exception otherwise.
72 :return: None or raises DbException on error
74 raise DbException("Method 'db_connect' not implemented")
76 def db_disconnect(self
):
78 Disconnect from database
83 def get_list(self
, table
, q_filter
=None):
85 Obtain a list of entries matching q_filter
86 :param table: collection or table
87 :param q_filter: Filter
88 :return: a list (can be empty) with the found entries. Raises DbException on error
90 raise DbException("Method 'get_list' not implemented")
92 def count(self
, table
, q_filter
=None):
94 Count the number of entries matching q_filter
95 :param table: collection or table
96 :param q_filter: Filter
97 :return: number of entries found (can be zero)
98 :raise: DbException on error
100 raise DbException("Method 'count' not implemented")
102 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
104 Obtain one entry matching q_filter
105 :param table: collection or table
106 :param q_filter: Filter
107 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
108 it raises a DbException
109 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
110 that it raises a DbException
111 :return: The requested element, or None
113 raise DbException("Method 'get_one' not implemented")
115 def del_list(self
, table
, q_filter
=None):
117 Deletes all entries that match q_filter
118 :param table: collection or table
119 :param q_filter: Filter
120 :return: Dict with the number of entries deleted
122 raise DbException("Method 'del_list' not implemented")
124 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
126 Deletes one entry that matches q_filter
127 :param table: collection or table
128 :param q_filter: Filter
129 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
130 which case it raises a DbException
131 :return: Dict with the number of entries deleted
133 raise DbException("Method 'del_one' not implemented")
135 def create(self
, table
, indata
):
137 Add a new entry at database
138 :param table: collection or table
139 :param indata: content to be added
140 :return: database '_id' of the inserted element. Raises a DbException on error
142 raise DbException("Method 'create' not implemented")
144 def create_list(self
, table
, indata_list
):
146 Add several entries at once
147 :param table: collection or table
148 :param indata_list: list of elements to insert. Each element must be a dictionary.
149 An '_id' key based on random uuid is added at each element if missing
150 :return: list of inserted '_id's. Exception on error
152 raise DbException("Method 'create_list' not implemented")
154 def set_one(self
, table
, q_filter
, update_dict
, fail_on_empty
=True, unset
=None, pull
=None, push
=None,
155 push_list
=None, pull_list
=None):
157 Modifies an entry at database
158 :param table: collection or table
159 :param q_filter: Filter
160 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
161 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
162 it raises a DbException
163 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
164 ignored. If not exist, it is ignored
165 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
166 if exist in the array is removed. If not exist, it is ignored
167 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
168 is appended to the end of the array
169 :param pull_list: Same as pull but values are arrays where each item is removed from the array
170 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
172 :return: Dict with the number of entries modified. None if no matching is found.
174 raise DbException("Method 'set_one' not implemented")
176 def set_list(self
, table
, q_filter
, update_dict
, unset
=None, pull
=None, push
=None, push_list
=None, pull_list
=None):
178 Modifies al matching entries at database
179 :param table: collection or table
180 :param q_filter: Filter
181 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
182 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
183 ignored. If not exist, it is ignored
184 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
185 if exist in the array is removed. If not exist, it is ignored
186 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
187 is appended to the end of the array
188 :param pull_list: Same as pull but values are arrays where each item is removed from the array
189 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
191 :return: Dict with the number of entries modified
193 raise DbException("Method 'set_list' not implemented")
195 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
197 Replace the content of an entry
198 :param table: collection or table
199 :param _id: internal database id
200 :param indata: content to replace
201 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
202 it raises a DbException
203 :return: Dict with the number of entries replaced
205 raise DbException("Method 'replace' not implemented")
207 def _join_secret_key(self
, update_key
):
209 Returns a xor byte combination of the internal secret_key and the provided update_key.
210 It does not modify the internal secret_key. Used for adding salt, join keys, etc.
211 :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
212 :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
216 return self
.secret_key
217 elif isinstance(update_key
, str):
218 update_key_bytes
= update_key
.encode()
220 update_key_bytes
= update_key
222 new_secret_key
= bytearray(self
.secret_key
) if self
.secret_key
else bytearray(32)
223 for i
, b
in enumerate(update_key_bytes
):
224 new_secret_key
[i
% 32] ^
= b
225 return bytes(new_secret_key
)
227 def set_secret_key(self
, new_secret_key
, replace
=False):
229 Updates internal secret_key used for encryption, with a byte xor
230 :param new_secret_key: string or byte array. It is recommended a 32 byte length
231 :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
235 self
.secret_key
= None
236 self
.secret_key
= self
._join
_secret
_key
(new_secret_key
)
238 def get_secret_key(self
):
240 Get the database secret key in case it is not done when "connect" is called. It can happens when database is
241 empty after an initial install. It should skip if secret is already obtained.
245 def encrypt(self
, value
, schema_version
=None, salt
=None):
248 :param value: value to be encrypted. It is string/unicode
249 :param schema_version: used for version control. If None or '1.0' no encryption is done.
250 If '1.1' symmetric AES encryption is done
251 :param salt: optional salt to be used. Must be str
252 :return: Encrypted content of value
254 self
.get_secret_key()
255 if not self
.secret_key
or not schema_version
or schema_version
== '1.0':
258 secret_key
= self
._join
_secret
_key
(salt
)
259 cipher
= AES
.new(secret_key
)
260 padded_private_msg
= value
+ ('\0' * ((16-len(value
)) % 16))
261 encrypted_msg
= cipher
.encrypt(padded_private_msg
)
262 encoded_encrypted_msg
= b64encode(encrypted_msg
)
263 return encoded_encrypted_msg
.decode("ascii")
265 def decrypt(self
, value
, schema_version
=None, salt
=None):
267 Decrypt an encrypted value
268 :param value: value to be decrypted. It is a base64 string
269 :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
270 If '1.1' symmetric AES encryption has been done
271 :param salt: optional salt to be used
272 :return: Plain content of value
274 self
.get_secret_key()
275 if not self
.secret_key
or not schema_version
or schema_version
== '1.0':
278 secret_key
= self
._join
_secret
_key
(salt
)
279 encrypted_msg
= b64decode(value
)
280 cipher
= AES
.new(secret_key
)
281 decrypted_msg
= cipher
.decrypt(encrypted_msg
)
283 unpadded_private_msg
= decrypted_msg
.decode().rstrip('\0')
284 except UnicodeDecodeError:
285 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
286 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
287 return unpadded_private_msg
289 def encrypt_decrypt_fields(self
, item
, action
, fields
=None, flags
=None, schema_version
=None, salt
=None):
292 self
.get_secret_key()
293 actions
= ['encrypt', 'decrypt']
294 if action
.lower() not in actions
:
295 raise DbException("Unknown action ({}): Must be one of {}".format(action
, actions
),
296 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
297 method
= self
.encrypt
if action
.lower() == 'encrypt' else self
.decrypt
302 if isinstance(_item
, list):
305 elif isinstance(_item
, dict):
306 for key
, val
in _item
.items():
307 if isinstance(val
, str):
308 if any(re
.search(f
, key
, flags
) for f
in fields
):
309 _item
[key
] = method(val
, schema_version
, salt
)
315 def deep_update_rfc7396(dict_to_change
, dict_reference
, key_list
=None):
317 Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
318 Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
319 It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
320 For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
321 $[index] <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
322 the element of the array is deleted, otherwise it is edited.
323 $+[index] The value is inserted at this <index>. A value of None has not sense and an exception is raised.
324 $+ The value is appended at the end. A value of None has not sense and an exception is raised.
325 $val It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
326 that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
327 Nothing happens if no match is found. If the value is None the matched elements are deleted.
328 $key: val In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
329 that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
330 format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
331 found. If value is None the matched items are deleted, otherwise they are edited.
332 $+val If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
333 is changed. A value of None has not sense.
334 $+key: val If no match if found (see '$key: val'), the value is appended to the array. If any match is found
335 nothing is changed. A value of None has not sense.
336 If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
337 order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
338 insertions must take into account the deleted items.
339 :param dict_to_change: Target dictionary to be changed.
340 :param dict_reference: Dictionary that contains changes to be applied.
341 :param key_list: This is used internally for recursive calls. Do not fill this parameter.
342 :return: none or raises and exception only at array modification when there is a bad format or conflict.
344 def _deep_update_array(array_to_change
, _dict_reference
, _key_list
):
346 to_insert_at_index
= {}
347 values_to_edit_delete
= {}
348 indexes_to_edit_delete
= []
351 for k
in _dict_reference
:
352 _key_list
[-1] = str(k
)
353 if not isinstance(k
, str) or not k
.startswith("$"):
354 if array_edition
is True:
355 raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
356 " same dict at '{}'".format(":".join(_key_list
[:-1])))
357 array_edition
= False
360 if array_edition
is False:
361 raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
362 " same dict at '{}'".format(":".join(_key_list
[:-1])))
365 indexes
= [] # indexes to edit or insert
367 if kitem
.startswith('+'):
370 if _dict_reference
[k
] is None:
371 raise DbException("A value of None has not sense for insertions at '{}'".format(
372 ":".join(_key_list
)))
374 if kitem
.startswith('[') and kitem
.endswith(']'):
376 index
= int(kitem
[1:-1])
378 index
+= len(array_to_change
)
380 index
= 0 # skip outside index edition
381 indexes
.append(index
)
383 raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
384 ":".join(_key_list
)))
386 # match_found_skip = False
388 filter_in
= yaml
.safe_load(kitem
)
390 raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list
)))
391 if isinstance(filter_in
, dict):
392 for index
, item
in enumerate(array_to_change
):
393 for filter_k
, filter_v
in filter_in
.items():
394 if not isinstance(item
, dict) or filter_k
not in item
or item
[filter_k
] != filter_v
:
398 # match_found_skip = True
402 indexes
.append(index
)
406 while True: # if not match a ValueError exception will be raise
407 index
= array_to_change
.index(filter_in
, index
)
409 # match_found_skip = True
412 indexes
.append(index
)
417 # if match_found_skip:
420 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
421 ":".join(_key_list
)))
422 for index
in indexes
:
424 if index
in to_insert_at_index
and to_insert_at_index
[index
] != _dict_reference
[k
]:
425 # Several different insertions on the same item of the array
426 raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
427 ":".join(_key_list
), index
))
428 to_insert_at_index
[index
] = _dict_reference
[k
]
430 if index
in indexes_to_edit_delete
and values_to_edit_delete
[index
] != _dict_reference
[k
]:
431 # Several different editions on the same item of the array
432 raise DbException("Conflict at '{}'. Several editions on array index {}".format(
433 ":".join(_key_list
), index
))
434 indexes_to_edit_delete
.append(index
)
435 values_to_edit_delete
[index
] = _dict_reference
[k
]
438 to_append
[k
] = _dict_reference
[k
]
439 # elif _dict_reference[k] is not None:
440 # raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
441 # ":".join(_key_list)))
443 # edition/deletion is done before insertion
444 indexes_to_edit_delete
.sort(reverse
=True)
445 for index
in indexes_to_edit_delete
:
446 _key_list
[-1] = str(index
)
448 if values_to_edit_delete
[index
] is None: # None->Anything
450 del (array_to_change
[index
])
452 pass # it is not consider an error if this index does not exist
453 elif not isinstance(values_to_edit_delete
[index
], dict): # NotDict->Anything
454 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
455 elif isinstance(array_to_change
[index
], dict): # Dict->Dict
456 deep_update_rfc7396(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
)
457 else: # Dict->NotDict
458 if isinstance(array_to_change
[index
], list): # Dict->List. Check extra array edition
459 if _deep_update_array(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
):
461 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
462 # calling deep_update_rfc7396 to delete the None values
463 deep_update_rfc7396(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
)
465 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list
)))
467 # insertion with indexes
468 to_insert_indexes
= list(to_insert_at_index
.keys())
469 to_insert_indexes
.sort(reverse
=True)
470 for index
in to_insert_indexes
:
471 array_to_change
.insert(index
, to_insert_at_index
[index
])
474 for k
, insert_value
in to_append
.items():
475 _key_list
[-1] = str(k
)
476 insert_value_copy
= deepcopy(insert_value
)
477 if isinstance(insert_value_copy
, dict):
478 # calling deep_update_rfc7396 to delete the None values
479 deep_update_rfc7396(insert_value_copy
, insert_value
, _key_list
)
480 array_to_change
.append(insert_value_copy
)
490 for k
in dict_reference
:
491 key_list
[-1] = str(k
)
492 if dict_reference
[k
] is None: # None->Anything
493 if k
in dict_to_change
:
494 del dict_to_change
[k
]
495 elif not isinstance(dict_reference
[k
], dict): # NotDict->Anything
496 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
497 elif k
not in dict_to_change
: # Dict->Empty
498 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
499 # calling deep_update_rfc7396 to delete the None values
500 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
501 elif isinstance(dict_to_change
[k
], dict): # Dict->Dict
502 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
503 else: # Dict->NotDict
504 if isinstance(dict_to_change
[k
], list): # Dict->List. Check extra array edition
505 if _deep_update_array(dict_to_change
[k
], dict_reference
[k
], key_list
):
507 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
508 # calling deep_update_rfc7396 to delete the None values
509 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
513 def deep_update(dict_to_change
, dict_reference
):
514 """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
515 return deep_update_rfc7396(dict_to_change
, dict_reference
)