1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
21 from http
import HTTPStatus
22 from copy
import deepcopy
23 from Crypto
.Cipher
import AES
24 from base64
import b64decode
, b64encode
25 from osm_common
.common_utils
import FakeLock
26 from threading
import Lock
28 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
31 class DbException(Exception):
33 def __init__(self
, message
, http_code
=HTTPStatus
.NOT_FOUND
):
34 self
.http_code
= http_code
35 Exception.__init
__(self
, "database exception " + str(message
))
40 def __init__(self
, logger_name
='db', lock
=False):
43 :param logger_name: logging name
44 :param lock: Used to protect simultaneous access to the same instance class by several threads:
45 False, None: Do not protect, this object will only be accessed by one thread
46 True: This object needs to be protected by several threads accessing.
47 Lock object. Use thi Lock for the threads access protection
49 self
.logger
= logging
.getLogger(logger_name
)
50 self
.secret_key
= None # 32 bytes length array used for encrypt/decrypt
52 self
.lock
= FakeLock()
55 elif isinstance(lock
, Lock
):
58 raise ValueError("lock parameter must be a Lock classclass or boolean")
60 def db_connect(self
, config
, target_version
=None):
63 :param config: Configuration of database. Contains among others:
64 host: database hosst (mandatory)
65 port: database port (mandatory)
66 name: database name (mandatory)
67 user: database username
68 password: database password
69 commonkey: common OSM key used for sensible information encryption
70 materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
71 :param target_version: if provided it checks if database contains required version, raising exception otherwise.
72 :return: None or raises DbException on error
74 raise DbException("Method 'db_connect' not implemented")
76 def db_disconnect(self
):
78 Disconnect from database
83 def get_list(self
, table
, q_filter
=None):
85 Obtain a list of entries matching q_filter
86 :param table: collection or table
87 :param q_filter: Filter
88 :return: a list (can be empty) with the found entries. Raises DbException on error
90 raise DbException("Method 'get_list' not implemented")
92 def count(self
, table
, q_filter
=None):
94 Count the number of entries matching q_filter
95 :param table: collection or table
96 :param q_filter: Filter
97 :return: number of entries found (can be zero)
98 :raise: DbException on error
100 raise DbException("Method 'count' not implemented")
102 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
104 Obtain one entry matching q_filter
105 :param table: collection or table
106 :param q_filter: Filter
107 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
108 it raises a DbException
109 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
110 that it raises a DbException
111 :return: The requested element, or None
113 raise DbException("Method 'get_one' not implemented")
115 def del_list(self
, table
, q_filter
=None):
117 Deletes all entries that match q_filter
118 :param table: collection or table
119 :param q_filter: Filter
120 :return: Dict with the number of entries deleted
122 raise DbException("Method 'del_list' not implemented")
124 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
126 Deletes one entry that matches q_filter
127 :param table: collection or table
128 :param q_filter: Filter
129 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
130 which case it raises a DbException
131 :return: Dict with the number of entries deleted
133 raise DbException("Method 'del_one' not implemented")
135 def create(self
, table
, indata
):
137 Add a new entry at database
138 :param table: collection or table
139 :param indata: content to be added
140 :return: database id of the inserted element. Raises a DbException on error
142 raise DbException("Method 'create' not implemented")
144 def set_one(self
, table
, q_filter
, update_dict
, fail_on_empty
=True, unset
=None, pull
=None, push
=None):
146 Modifies an entry at database
147 :param table: collection or table
148 :param q_filter: Filter
149 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
150 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
151 it raises a DbException
152 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
153 ignored. If not exist, it is ignored
154 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
155 if exist in the array is removed. If not exist, it is ignored
156 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
157 is appended to the end of the array
158 :return: Dict with the number of entries modified. None if no matching is found.
160 raise DbException("Method 'set_one' not implemented")
162 def set_list(self
, table
, q_filter
, update_dict
):
164 Modifies al matching entries at database
165 :param table: collection or table
166 :param q_filter: Filter
167 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
168 :return: Dict with the number of entries modified
170 raise DbException("Method 'set_list' not implemented")
172 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
174 Replace the content of an entry
175 :param table: collection or table
176 :param _id: internal database id
177 :param indata: content to replace
178 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
179 it raises a DbException
180 :return: Dict with the number of entries replaced
182 raise DbException("Method 'replace' not implemented")
184 def _join_secret_key(self
, update_key
):
186 Returns a xor byte combination of the internal secret_key and the provided update_key.
187 It does not modify the internal secret_key. Used for adding salt, join keys, etc.
188 :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
189 :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
193 return self
.secret_key
194 elif isinstance(update_key
, str):
195 update_key_bytes
= update_key
.encode()
197 update_key_bytes
= update_key
199 new_secret_key
= bytearray(self
.secret_key
) if self
.secret_key
else bytearray(32)
200 for i
, b
in enumerate(update_key_bytes
):
201 new_secret_key
[i
% 32] ^
= b
202 return bytes(new_secret_key
)
204 def set_secret_key(self
, new_secret_key
, replace
=False):
206 Updates internal secret_key used for encryption, with a byte xor
207 :param new_secret_key: string or byte array. It is recommended a 32 byte length
208 :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
212 self
.secret_key
= None
213 self
.secret_key
= self
._join
_secret
_key
(new_secret_key
)
215 def encrypt(self
, value
, schema_version
=None, salt
=None):
218 :param value: value to be encrypted. It is string/unicode
219 :param schema_version: used for version control. If None or '1.0' no encryption is done.
220 If '1.1' symmetric AES encryption is done
221 :param salt: optional salt to be used. Must be str
222 :return: Encrypted content of value
224 if not self
.secret_key
or not schema_version
or schema_version
== '1.0':
227 secret_key
= self
._join
_secret
_key
(salt
)
228 cipher
= AES
.new(secret_key
)
229 padded_private_msg
= value
+ ('\0' * ((16-len(value
)) % 16))
230 encrypted_msg
= cipher
.encrypt(padded_private_msg
)
231 encoded_encrypted_msg
= b64encode(encrypted_msg
)
232 return encoded_encrypted_msg
.decode("ascii")
234 def decrypt(self
, value
, schema_version
=None, salt
=None):
236 Decrypt an encrypted value
237 :param value: value to be decrypted. It is a base64 string
238 :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
239 If '1.1' symmetric AES encryption has been done
240 :param salt: optional salt to be used
241 :return: Plain content of value
243 if not self
.secret_key
or not schema_version
or schema_version
== '1.0':
246 secret_key
= self
._join
_secret
_key
(salt
)
247 encrypted_msg
= b64decode(value
)
248 cipher
= AES
.new(secret_key
)
249 decrypted_msg
= cipher
.decrypt(encrypted_msg
)
251 unpadded_private_msg
= decrypted_msg
.decode().rstrip('\0')
252 except UnicodeDecodeError:
253 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
254 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
255 return unpadded_private_msg
257 def encrypt_decrypt_fields(self
, item
, action
, fields
=None, flags
=re
.I
, schema_version
=None, salt
=None):
260 actions
= ['encrypt', 'decrypt']
261 if action
.lower() not in actions
:
262 raise DbException("Unknown action ({}): Must be one of {}".format(action
, actions
),
263 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
264 method
= self
.encrypt
if action
.lower() == 'encrypt' else self
.decrypt
267 if isinstance(item
, list):
270 elif isinstance(item
, dict):
271 for key
, val
in item
.items():
272 if any(re
.search(f
, key
, flags
) for f
in fields
) and isinstance(val
, str):
273 item
[key
] = method(val
, schema_version
, salt
)
280 def deep_update_rfc7396(dict_to_change
, dict_reference
, key_list
=None):
282 Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
283 Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
284 It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
285 For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
286 $[index] <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
287 the element of the array is deleted, otherwise it is edited.
288 $+[index] The value is inserted at this <index>. A value of None has not sense and an exception is raised.
289 $+ The value is appended at the end. A value of None has not sense and an exception is raised.
290 $val It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
291 that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
292 Nothing happens if no match is found. If the value is None the matched elements are deleted.
293 $key: val In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
294 that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
295 format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
296 found. If value is None the matched items are deleted, otherwise they are edited.
297 $+val If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
298 is changed. A value of None has not sense.
299 $+key: val If no match if found (see '$key: val'), the value is appended to the array. If any match is found
300 nothing is changed. A value of None has not sense.
301 If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
302 order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
303 insertions must take into account the deleted items.
304 :param dict_to_change: Target dictionary to be changed.
305 :param dict_reference: Dictionary that contains changes to be applied.
306 :param key_list: This is used internally for recursive calls. Do not fill this parameter.
307 :return: none or raises and exception only at array modification when there is a bad format or conflict.
309 def _deep_update_array(array_to_change
, _dict_reference
, _key_list
):
311 to_insert_at_index
= {}
312 values_to_edit_delete
= {}
313 indexes_to_edit_delete
= []
316 for k
in _dict_reference
:
317 _key_list
[-1] = str(k
)
318 if not isinstance(k
, str) or not k
.startswith("$"):
319 if array_edition
is True:
320 raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
321 " same dict at '{}'".format(":".join(_key_list
[:-1])))
322 array_edition
= False
325 if array_edition
is False:
326 raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
327 " same dict at '{}'".format(":".join(_key_list
[:-1])))
330 indexes
= [] # indexes to edit or insert
332 if kitem
.startswith('+'):
335 if _dict_reference
[k
] is None:
336 raise DbException("A value of None has not sense for insertions at '{}'".format(
337 ":".join(_key_list
)))
339 if kitem
.startswith('[') and kitem
.endswith(']'):
341 index
= int(kitem
[1:-1])
343 index
+= len(array_to_change
)
345 index
= 0 # skip outside index edition
346 indexes
.append(index
)
348 raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
349 ":".join(_key_list
)))
351 # match_found_skip = False
353 filter_in
= yaml
.safe_load(kitem
)
355 raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list
)))
356 if isinstance(filter_in
, dict):
357 for index
, item
in enumerate(array_to_change
):
358 for filter_k
, filter_v
in filter_in
.items():
359 if not isinstance(item
, dict) or filter_k
not in item
or item
[filter_k
] != filter_v
:
363 # match_found_skip = True
367 indexes
.append(index
)
371 while True: # if not match a ValueError exception will be raise
372 index
= array_to_change
.index(filter_in
, index
)
374 # match_found_skip = True
377 indexes
.append(index
)
382 # if match_found_skip:
385 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
386 ":".join(_key_list
)))
387 for index
in indexes
:
389 if index
in to_insert_at_index
and to_insert_at_index
[index
] != _dict_reference
[k
]:
390 # Several different insertions on the same item of the array
391 raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
392 ":".join(_key_list
), index
))
393 to_insert_at_index
[index
] = _dict_reference
[k
]
395 if index
in indexes_to_edit_delete
and values_to_edit_delete
[index
] != _dict_reference
[k
]:
396 # Several different editions on the same item of the array
397 raise DbException("Conflict at '{}'. Several editions on array index {}".format(
398 ":".join(_key_list
), index
))
399 indexes_to_edit_delete
.append(index
)
400 values_to_edit_delete
[index
] = _dict_reference
[k
]
403 to_append
[k
] = _dict_reference
[k
]
404 # elif _dict_reference[k] is not None:
405 # raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
406 # ":".join(_key_list)))
408 # edition/deletion is done before insertion
409 indexes_to_edit_delete
.sort(reverse
=True)
410 for index
in indexes_to_edit_delete
:
411 _key_list
[-1] = str(index
)
413 if values_to_edit_delete
[index
] is None: # None->Anything
415 del (array_to_change
[index
])
417 pass # it is not consider an error if this index does not exist
418 elif not isinstance(values_to_edit_delete
[index
], dict): # NotDict->Anything
419 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
420 elif isinstance(array_to_change
[index
], dict): # Dict->Dict
421 deep_update_rfc7396(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
)
422 else: # Dict->NotDict
423 if isinstance(array_to_change
[index
], list): # Dict->List. Check extra array edition
424 if _deep_update_array(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
):
426 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
427 # calling deep_update_rfc7396 to delete the None values
428 deep_update_rfc7396(array_to_change
[index
], values_to_edit_delete
[index
], _key_list
)
430 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list
)))
432 # insertion with indexes
433 to_insert_indexes
= list(to_insert_at_index
.keys())
434 to_insert_indexes
.sort(reverse
=True)
435 for index
in to_insert_indexes
:
436 array_to_change
.insert(index
, to_insert_at_index
[index
])
439 for k
, insert_value
in to_append
.items():
440 _key_list
[-1] = str(k
)
441 insert_value_copy
= deepcopy(insert_value
)
442 if isinstance(insert_value_copy
, dict):
443 # calling deep_update_rfc7396 to delete the None values
444 deep_update_rfc7396(insert_value_copy
, insert_value
, _key_list
)
445 array_to_change
.append(insert_value_copy
)
455 for k
in dict_reference
:
456 key_list
[-1] = str(k
)
457 if dict_reference
[k
] is None: # None->Anything
458 if k
in dict_to_change
:
459 del dict_to_change
[k
]
460 elif not isinstance(dict_reference
[k
], dict): # NotDict->Anything
461 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
462 elif k
not in dict_to_change
: # Dict->Empty
463 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
464 # calling deep_update_rfc7396 to delete the None values
465 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
466 elif isinstance(dict_to_change
[k
], dict): # Dict->Dict
467 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
468 else: # Dict->NotDict
469 if isinstance(dict_to_change
[k
], list): # Dict->List. Check extra array edition
470 if _deep_update_array(dict_to_change
[k
], dict_reference
[k
], key_list
):
472 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
473 # calling deep_update_rfc7396 to delete the None values
474 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
478 def deep_update(dict_to_change
, dict_reference
):
479 """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
480 return deep_update_rfc7396(dict_to_change
, dict_reference
)