1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 from base64
import b64decode
, b64encode
19 from copy
import deepcopy
20 from http
import HTTPStatus
23 from threading
import Lock
25 from Crypto
.Cipher
import AES
26 from osm_common
.common_utils
import FakeLock
29 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
32 class DbException(Exception):
33 def __init__(self
, message
, http_code
=HTTPStatus
.NOT_FOUND
):
34 self
.http_code
= http_code
35 Exception.__init
__(self
, "database exception " + str(message
))
39 def __init__(self
, logger_name
="db", lock
=False):
42 :param logger_name: logging name
43 :param lock: Used to protect simultaneous access to the same instance class by several threads:
44 False, None: Do not protect, this object will only be accessed by one thread
45 True: This object needs to be protected by several threads accessing.
46 Lock object. Use thi Lock for the threads access protection
48 self
.logger
= logging
.getLogger(logger_name
)
49 self
.secret_key
= None # 32 bytes length array used for encrypt/decrypt
51 self
.lock
= FakeLock()
54 elif isinstance(lock
, Lock
):
57 raise ValueError("lock parameter must be a Lock classclass or boolean")
59 def db_connect(self
, config
, target_version
=None):
62 :param config: Configuration of database. Contains among others:
63 host: database host (mandatory)
64 port: database port (mandatory)
65 name: database name (mandatory)
66 user: database username
67 password: database password
68 commonkey: common OSM key used for sensible information encryption
69 materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
70 :param target_version: if provided it checks if database contains required version, raising exception otherwise.
71 :return: None or raises DbException on error
73 raise DbException("Method 'db_connect' not implemented")
75 def db_disconnect(self
):
77 Disconnect from database
82 def get_list(self
, table
, q_filter
=None):
84 Obtain a list of entries matching q_filter
85 :param table: collection or table
86 :param q_filter: Filter
87 :return: a list (can be empty) with the found entries. Raises DbException on error
89 raise DbException("Method 'get_list' not implemented")
91 def count(self
, table
, q_filter
=None):
93 Count the number of entries matching q_filter
94 :param table: collection or table
95 :param q_filter: Filter
96 :return: number of entries found (can be zero)
97 :raise: DbException on error
99 raise DbException("Method 'count' not implemented")
101 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
103 Obtain one entry matching q_filter
104 :param table: collection or table
105 :param q_filter: Filter
106 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
107 it raises a DbException
108 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
109 that it raises a DbException
110 :return: The requested element, or None
112 raise DbException("Method 'get_one' not implemented")
114 def del_list(self
, table
, q_filter
=None):
116 Deletes all entries that match q_filter
117 :param table: collection or table
118 :param q_filter: Filter
119 :return: Dict with the number of entries deleted
121 raise DbException("Method 'del_list' not implemented")
123 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
125 Deletes one entry that matches q_filter
126 :param table: collection or table
127 :param q_filter: Filter
128 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
129 which case it raises a DbException
130 :return: Dict with the number of entries deleted
132 raise DbException("Method 'del_one' not implemented")
134 def create(self
, table
, indata
):
136 Add a new entry at database
137 :param table: collection or table
138 :param indata: content to be added
139 :return: database '_id' of the inserted element. Raises a DbException on error
141 raise DbException("Method 'create' not implemented")
143 def create_list(self
, table
, indata_list
):
145 Add several entries at once
146 :param table: collection or table
147 :param indata_list: list of elements to insert. Each element must be a dictionary.
148 An '_id' key based on random uuid is added at each element if missing
149 :return: list of inserted '_id's. Exception on error
151 raise DbException("Method 'create_list' not implemented")
166 Modifies an entry at database
167 :param table: collection or table
168 :param q_filter: Filter
169 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
170 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
171 it raises a DbException
172 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
173 ignored. If not exist, it is ignored
174 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
175 if exist in the array is removed. If not exist, it is ignored
176 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
177 is appended to the end of the array
178 :param pull_list: Same as pull but values are arrays where each item is removed from the array
179 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
181 :return: Dict with the number of entries modified. None if no matching is found.
183 raise DbException("Method 'set_one' not implemented")
197 Modifies al matching entries at database
198 :param table: collection or table
199 :param q_filter: Filter
200 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
201 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
202 ignored. If not exist, it is ignored
203 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
204 if exist in the array is removed. If not exist, it is ignored
205 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
206 is appended to the end of the array
207 :param pull_list: Same as pull but values are arrays where each item is removed from the array
208 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
210 :return: Dict with the number of entries modified
212 raise DbException("Method 'set_list' not implemented")
214 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
216 Replace the content of an entry
217 :param table: collection or table
218 :param _id: internal database id
219 :param indata: content to replace
220 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
221 it raises a DbException
222 :return: Dict with the number of entries replaced
224 raise DbException("Method 'replace' not implemented")
226 def _join_secret_key(self
, update_key
):
228 Returns a xor byte combination of the internal secret_key and the provided update_key.
229 It does not modify the internal secret_key. Used for adding salt, join keys, etc.
230 :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
231 :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
235 return self
.secret_key
236 elif isinstance(update_key
, str):
237 update_key_bytes
= update_key
.encode()
239 update_key_bytes
= update_key
242 bytearray(self
.secret_key
) if self
.secret_key
else bytearray(32)
244 for i
, b
in enumerate(update_key_bytes
):
245 new_secret_key
[i
% 32] ^
= b
246 return bytes(new_secret_key
)
248 def set_secret_key(self
, new_secret_key
, replace
=False):
250 Updates internal secret_key used for encryption, with a byte xor
251 :param new_secret_key: string or byte array. It is recommended a 32 byte length
252 :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
256 self
.secret_key
= None
257 self
.secret_key
= self
._join
_secret
_key
(new_secret_key
)
259 def get_secret_key(self
):
261 Get the database secret key in case it is not done when "connect" is called. It can happens when database is
262 empty after an initial install. It should skip if secret is already obtained.
266 def encrypt(self
, value
, schema_version
=None, salt
=None):
269 :param value: value to be encrypted. It is string/unicode
270 :param schema_version: used for version control. If None or '1.0' no encryption is done.
271 If '1.1' symmetric AES encryption is done
272 :param salt: optional salt to be used. Must be str
273 :return: Encrypted content of value
275 self
.get_secret_key()
276 if not self
.secret_key
or not schema_version
or schema_version
== "1.0":
279 secret_key
= self
._join
_secret
_key
(salt
)
280 cipher
= AES
.new(secret_key
)
281 padded_private_msg
= value
+ ("\0" * ((16 - len(value
)) % 16))
282 encrypted_msg
= cipher
.encrypt(padded_private_msg
)
283 encoded_encrypted_msg
= b64encode(encrypted_msg
)
284 return encoded_encrypted_msg
.decode("ascii")
286 def decrypt(self
, value
, schema_version
=None, salt
=None):
288 Decrypt an encrypted value
289 :param value: value to be decrypted. It is a base64 string
290 :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
291 If '1.1' symmetric AES encryption has been done
292 :param salt: optional salt to be used
293 :return: Plain content of value
295 self
.get_secret_key()
296 if not self
.secret_key
or not schema_version
or schema_version
== "1.0":
299 secret_key
= self
._join
_secret
_key
(salt
)
300 encrypted_msg
= b64decode(value
)
301 cipher
= AES
.new(secret_key
)
302 decrypted_msg
= cipher
.decrypt(encrypted_msg
)
304 unpadded_private_msg
= decrypted_msg
.decode().rstrip("\0")
305 except UnicodeDecodeError:
307 "Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
308 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
,
310 return unpadded_private_msg
312 def encrypt_decrypt_fields(
313 self
, item
, action
, fields
=None, flags
=None, schema_version
=None, salt
=None
317 self
.get_secret_key()
318 actions
= ["encrypt", "decrypt"]
319 if action
.lower() not in actions
:
321 "Unknown action ({}): Must be one of {}".format(action
, actions
),
322 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
,
324 method
= self
.encrypt
if action
.lower() == "encrypt" else self
.decrypt
329 if isinstance(_item
, list):
332 elif isinstance(_item
, dict):
333 for key
, val
in _item
.items():
334 if isinstance(val
, str):
335 if any(re
.search(f
, key
, flags
) for f
in fields
):
336 _item
[key
] = method(val
, schema_version
, salt
)
343 def deep_update_rfc7396(dict_to_change
, dict_reference
, key_list
=None):
345 Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
346 Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
347 It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
348 For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
349 $[index] <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
350 the element of the array is deleted, otherwise it is edited.
351 $+[index] The value is inserted at this <index>. A value of None has not sense and an exception is raised.
352 $+ The value is appended at the end. A value of None has not sense and an exception is raised.
353 $val It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
354 that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
355 Nothing happens if no match is found. If the value is None the matched elements are deleted.
356 $key: val In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
357 that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
358 format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
359 found. If value is None the matched items are deleted, otherwise they are edited.
360 $+val If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
361 is changed. A value of None has not sense.
362 $+key: val If no match if found (see '$key: val'), the value is appended to the array. If any match is found
363 nothing is changed. A value of None has not sense.
364 If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
365 order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
366 insertions must take into account the deleted items.
367 :param dict_to_change: Target dictionary to be changed.
368 :param dict_reference: Dictionary that contains changes to be applied.
369 :param key_list: This is used internally for recursive calls. Do not fill this parameter.
370 :return: none or raises and exception only at array modification when there is a bad format or conflict.
373 def _deep_update_array(array_to_change
, _dict_reference
, _key_list
):
375 to_insert_at_index
= {}
376 values_to_edit_delete
= {}
377 indexes_to_edit_delete
= []
380 for k
in _dict_reference
:
381 _key_list
[-1] = str(k
)
382 if not isinstance(k
, str) or not k
.startswith("$"):
383 if array_edition
is True:
385 "Found array edition (keys starting with '$') and pure dictionary edition in the"
386 " same dict at '{}'".format(":".join(_key_list
[:-1]))
388 array_edition
= False
391 if array_edition
is False:
393 "Found array edition (keys starting with '$') and pure dictionary edition in the"
394 " same dict at '{}'".format(":".join(_key_list
[:-1]))
398 indexes
= [] # indexes to edit or insert
400 if kitem
.startswith("+"):
403 if _dict_reference
[k
] is None:
405 "A value of None has not sense for insertions at '{}'".format(
410 if kitem
.startswith("[") and kitem
.endswith("]"):
412 index
= int(kitem
[1:-1])
414 index
+= len(array_to_change
)
416 index
= 0 # skip outside index edition
417 indexes
.append(index
)
420 "Wrong format at '{}'. Expecting integer index inside quotes".format(
425 # match_found_skip = False
427 filter_in
= yaml
.safe_load(kitem
)
430 "Wrong format at '{}'. Expecting '$<yaml-format>'".format(
434 if isinstance(filter_in
, dict):
435 for index
, item
in enumerate(array_to_change
):
436 for filter_k
, filter_v
in filter_in
.items():
438 not isinstance(item
, dict)
439 or filter_k
not in item
440 or item
[filter_k
] != filter_v
445 # match_found_skip = True
449 indexes
.append(index
)
453 while True: # if not match a ValueError exception will be raise
454 index
= array_to_change
.index(filter_in
, index
)
456 # match_found_skip = True
459 indexes
.append(index
)
464 # if match_found_skip:
468 "Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
472 for index
in indexes
:
475 index
in to_insert_at_index
476 and to_insert_at_index
[index
] != _dict_reference
[k
]
478 # Several different insertions on the same item of the array
480 "Conflict at '{}'. Several insertions on same array index {}".format(
481 ":".join(_key_list
), index
484 to_insert_at_index
[index
] = _dict_reference
[k
]
487 index
in indexes_to_edit_delete
488 and values_to_edit_delete
[index
] != _dict_reference
[k
]
490 # Several different editions on the same item of the array
492 "Conflict at '{}'. Several editions on array index {}".format(
493 ":".join(_key_list
), index
496 indexes_to_edit_delete
.append(index
)
497 values_to_edit_delete
[index
] = _dict_reference
[k
]
500 to_append
[k
] = _dict_reference
[k
]
501 # elif _dict_reference[k] is not None:
502 # raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
503 # ":".join(_key_list)))
505 # edition/deletion is done before insertion
506 indexes_to_edit_delete
.sort(reverse
=True)
507 for index
in indexes_to_edit_delete
:
508 _key_list
[-1] = str(index
)
510 if values_to_edit_delete
[index
] is None: # None->Anything
512 del array_to_change
[index
]
514 pass # it is not consider an error if this index does not exist
516 values_to_edit_delete
[index
], dict
517 ): # NotDict->Anything
518 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
519 elif isinstance(array_to_change
[index
], dict): # Dict->Dict
521 array_to_change
[index
], values_to_edit_delete
[index
], _key_list
523 else: # Dict->NotDict
525 array_to_change
[index
], list
526 ): # Dict->List. Check extra array edition
527 if _deep_update_array(
528 array_to_change
[index
],
529 values_to_edit_delete
[index
],
533 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
534 # calling deep_update_rfc7396 to delete the None values
536 array_to_change
[index
], values_to_edit_delete
[index
], _key_list
540 "Array edition index out of range at '{}'".format(
545 # insertion with indexes
546 to_insert_indexes
= list(to_insert_at_index
.keys())
547 to_insert_indexes
.sort(reverse
=True)
548 for index
in to_insert_indexes
:
549 array_to_change
.insert(index
, to_insert_at_index
[index
])
552 for k
, insert_value
in to_append
.items():
553 _key_list
[-1] = str(k
)
554 insert_value_copy
= deepcopy(insert_value
)
555 if isinstance(insert_value_copy
, dict):
556 # calling deep_update_rfc7396 to delete the None values
557 deep_update_rfc7396(insert_value_copy
, insert_value
, _key_list
)
558 array_to_change
.append(insert_value_copy
)
568 for k
in dict_reference
:
569 key_list
[-1] = str(k
)
570 if dict_reference
[k
] is None: # None->Anything
571 if k
in dict_to_change
:
572 del dict_to_change
[k
]
573 elif not isinstance(dict_reference
[k
], dict): # NotDict->Anything
574 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
575 elif k
not in dict_to_change
: # Dict->Empty
576 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
577 # calling deep_update_rfc7396 to delete the None values
578 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
579 elif isinstance(dict_to_change
[k
], dict): # Dict->Dict
580 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
581 else: # Dict->NotDict
583 dict_to_change
[k
], list
584 ): # Dict->List. Check extra array edition
585 if _deep_update_array(dict_to_change
[k
], dict_reference
[k
], key_list
):
587 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
588 # calling deep_update_rfc7396 to delete the None values
589 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
593 def deep_update(dict_to_change
, dict_reference
):
594 """Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
595 return deep_update_rfc7396(dict_to_change
, dict_reference
)