1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Telefonica S.A.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
21 from http
import HTTPStatus
22 from copy
import deepcopy
23 from Crypto
.Cipher
import AES
24 from base64
import b64decode
, b64encode
25 from osm_common
.common_utils
import FakeLock
26 from threading
import Lock
28 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
31 class DbException(Exception):
32 def __init__(self
, message
, http_code
=HTTPStatus
.NOT_FOUND
):
33 self
.http_code
= http_code
34 Exception.__init
__(self
, "database exception " + str(message
))
38 def __init__(self
, logger_name
="db", lock
=False):
41 :param logger_name: logging name
42 :param lock: Used to protect simultaneous access to the same instance class by several threads:
43 False, None: Do not protect, this object will only be accessed by one thread
44 True: This object needs to be protected by several threads accessing.
45 Lock object. Use thi Lock for the threads access protection
47 self
.logger
= logging
.getLogger(logger_name
)
48 self
.secret_key
= None # 32 bytes length array used for encrypt/decrypt
50 self
.lock
= FakeLock()
53 elif isinstance(lock
, Lock
):
56 raise ValueError("lock parameter must be a Lock classclass or boolean")
58 def db_connect(self
, config
, target_version
=None):
61 :param config: Configuration of database. Contains among others:
62 host: database host (mandatory)
63 port: database port (mandatory)
64 name: database name (mandatory)
65 user: database username
66 password: database password
67 commonkey: common OSM key used for sensible information encryption
68 materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
69 :param target_version: if provided it checks if database contains required version, raising exception otherwise.
70 :return: None or raises DbException on error
72 raise DbException("Method 'db_connect' not implemented")
74 def db_disconnect(self
):
76 Disconnect from database
81 def get_list(self
, table
, q_filter
=None):
83 Obtain a list of entries matching q_filter
84 :param table: collection or table
85 :param q_filter: Filter
86 :return: a list (can be empty) with the found entries. Raises DbException on error
88 raise DbException("Method 'get_list' not implemented")
90 def count(self
, table
, q_filter
=None):
92 Count the number of entries matching q_filter
93 :param table: collection or table
94 :param q_filter: Filter
95 :return: number of entries found (can be zero)
96 :raise: DbException on error
98 raise DbException("Method 'count' not implemented")
100 def get_one(self
, table
, q_filter
=None, fail_on_empty
=True, fail_on_more
=True):
102 Obtain one entry matching q_filter
103 :param table: collection or table
104 :param q_filter: Filter
105 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
106 it raises a DbException
107 :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
108 that it raises a DbException
109 :return: The requested element, or None
111 raise DbException("Method 'get_one' not implemented")
113 def del_list(self
, table
, q_filter
=None):
115 Deletes all entries that match q_filter
116 :param table: collection or table
117 :param q_filter: Filter
118 :return: Dict with the number of entries deleted
120 raise DbException("Method 'del_list' not implemented")
122 def del_one(self
, table
, q_filter
=None, fail_on_empty
=True):
124 Deletes one entry that matches q_filter
125 :param table: collection or table
126 :param q_filter: Filter
127 :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
128 which case it raises a DbException
129 :return: Dict with the number of entries deleted
131 raise DbException("Method 'del_one' not implemented")
133 def create(self
, table
, indata
):
135 Add a new entry at database
136 :param table: collection or table
137 :param indata: content to be added
138 :return: database '_id' of the inserted element. Raises a DbException on error
140 raise DbException("Method 'create' not implemented")
142 def create_list(self
, table
, indata_list
):
144 Add several entries at once
145 :param table: collection or table
146 :param indata_list: list of elements to insert. Each element must be a dictionary.
147 An '_id' key based on random uuid is added at each element if missing
148 :return: list of inserted '_id's. Exception on error
150 raise DbException("Method 'create_list' not implemented")
165 Modifies an entry at database
166 :param table: collection or table
167 :param q_filter: Filter
168 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
169 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
170 it raises a DbException
171 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
172 ignored. If not exist, it is ignored
173 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
174 if exist in the array is removed. If not exist, it is ignored
175 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
176 is appended to the end of the array
177 :param pull_list: Same as pull but values are arrays where each item is removed from the array
178 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
180 :return: Dict with the number of entries modified. None if no matching is found.
182 raise DbException("Method 'set_one' not implemented")
196 Modifies al matching entries at database
197 :param table: collection or table
198 :param q_filter: Filter
199 :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
200 :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
201 ignored. If not exist, it is ignored
202 :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
203 if exist in the array is removed. If not exist, it is ignored
204 :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
205 is appended to the end of the array
206 :param pull_list: Same as pull but values are arrays where each item is removed from the array
207 :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
209 :return: Dict with the number of entries modified
211 raise DbException("Method 'set_list' not implemented")
213 def replace(self
, table
, _id
, indata
, fail_on_empty
=True):
215 Replace the content of an entry
216 :param table: collection or table
217 :param _id: internal database id
218 :param indata: content to replace
219 :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
220 it raises a DbException
221 :return: Dict with the number of entries replaced
223 raise DbException("Method 'replace' not implemented")
225 def _join_secret_key(self
, update_key
):
227 Returns a xor byte combination of the internal secret_key and the provided update_key.
228 It does not modify the internal secret_key. Used for adding salt, join keys, etc.
229 :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
230 :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
234 return self
.secret_key
235 elif isinstance(update_key
, str):
236 update_key_bytes
= update_key
.encode()
238 update_key_bytes
= update_key
241 bytearray(self
.secret_key
) if self
.secret_key
else bytearray(32)
243 for i
, b
in enumerate(update_key_bytes
):
244 new_secret_key
[i
% 32] ^
= b
245 return bytes(new_secret_key
)
247 def set_secret_key(self
, new_secret_key
, replace
=False):
249 Updates internal secret_key used for encryption, with a byte xor
250 :param new_secret_key: string or byte array. It is recommended a 32 byte length
251 :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
255 self
.secret_key
= None
256 self
.secret_key
= self
._join
_secret
_key
(new_secret_key
)
258 def get_secret_key(self
):
260 Get the database secret key in case it is not done when "connect" is called. It can happens when database is
261 empty after an initial install. It should skip if secret is already obtained.
265 def encrypt(self
, value
, schema_version
=None, salt
=None):
268 :param value: value to be encrypted. It is string/unicode
269 :param schema_version: used for version control. If None or '1.0' no encryption is done.
270 If '1.1' symmetric AES encryption is done
271 :param salt: optional salt to be used. Must be str
272 :return: Encrypted content of value
274 self
.get_secret_key()
275 if not self
.secret_key
or not schema_version
or schema_version
== "1.0":
278 secret_key
= self
._join
_secret
_key
(salt
)
279 cipher
= AES
.new(secret_key
)
280 padded_private_msg
= value
+ ("\0" * ((16 - len(value
)) % 16))
281 encrypted_msg
= cipher
.encrypt(padded_private_msg
)
282 encoded_encrypted_msg
= b64encode(encrypted_msg
)
283 return encoded_encrypted_msg
.decode("ascii")
285 def decrypt(self
, value
, schema_version
=None, salt
=None):
287 Decrypt an encrypted value
288 :param value: value to be decrypted. It is a base64 string
289 :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
290 If '1.1' symmetric AES encryption has been done
291 :param salt: optional salt to be used
292 :return: Plain content of value
294 self
.get_secret_key()
295 if not self
.secret_key
or not schema_version
or schema_version
== "1.0":
298 secret_key
= self
._join
_secret
_key
(salt
)
299 encrypted_msg
= b64decode(value
)
300 cipher
= AES
.new(secret_key
)
301 decrypted_msg
= cipher
.decrypt(encrypted_msg
)
303 unpadded_private_msg
= decrypted_msg
.decode().rstrip("\0")
304 except UnicodeDecodeError:
306 "Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
307 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
,
309 return unpadded_private_msg
311 def encrypt_decrypt_fields(
312 self
, item
, action
, fields
=None, flags
=None, schema_version
=None, salt
=None
316 self
.get_secret_key()
317 actions
= ["encrypt", "decrypt"]
318 if action
.lower() not in actions
:
320 "Unknown action ({}): Must be one of {}".format(action
, actions
),
321 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
,
323 method
= self
.encrypt
if action
.lower() == "encrypt" else self
.decrypt
328 if isinstance(_item
, list):
331 elif isinstance(_item
, dict):
332 for key
, val
in _item
.items():
333 if isinstance(val
, str):
334 if any(re
.search(f
, key
, flags
) for f
in fields
):
335 _item
[key
] = method(val
, schema_version
, salt
)
342 def deep_update_rfc7396(dict_to_change
, dict_reference
, key_list
=None):
344 Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
345 Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
346 It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
347 For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
348 $[index] <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
349 the element of the array is deleted, otherwise it is edited.
350 $+[index] The value is inserted at this <index>. A value of None has not sense and an exception is raised.
351 $+ The value is appended at the end. A value of None has not sense and an exception is raised.
352 $val It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
353 that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
354 Nothing happens if no match is found. If the value is None the matched elements are deleted.
355 $key: val In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
356 that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
357 format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
358 found. If value is None the matched items are deleted, otherwise they are edited.
359 $+val If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
360 is changed. A value of None has not sense.
361 $+key: val If no match if found (see '$key: val'), the value is appended to the array. If any match is found
362 nothing is changed. A value of None has not sense.
363 If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
364 order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
365 insertions must take into account the deleted items.
366 :param dict_to_change: Target dictionary to be changed.
367 :param dict_reference: Dictionary that contains changes to be applied.
368 :param key_list: This is used internally for recursive calls. Do not fill this parameter.
369 :return: none or raises and exception only at array modification when there is a bad format or conflict.
372 def _deep_update_array(array_to_change
, _dict_reference
, _key_list
):
374 to_insert_at_index
= {}
375 values_to_edit_delete
= {}
376 indexes_to_edit_delete
= []
379 for k
in _dict_reference
:
380 _key_list
[-1] = str(k
)
381 if not isinstance(k
, str) or not k
.startswith("$"):
382 if array_edition
is True:
384 "Found array edition (keys starting with '$') and pure dictionary edition in the"
385 " same dict at '{}'".format(":".join(_key_list
[:-1]))
387 array_edition
= False
390 if array_edition
is False:
392 "Found array edition (keys starting with '$') and pure dictionary edition in the"
393 " same dict at '{}'".format(":".join(_key_list
[:-1]))
397 indexes
= [] # indexes to edit or insert
399 if kitem
.startswith("+"):
402 if _dict_reference
[k
] is None:
404 "A value of None has not sense for insertions at '{}'".format(
409 if kitem
.startswith("[") and kitem
.endswith("]"):
411 index
= int(kitem
[1:-1])
413 index
+= len(array_to_change
)
415 index
= 0 # skip outside index edition
416 indexes
.append(index
)
419 "Wrong format at '{}'. Expecting integer index inside quotes".format(
424 # match_found_skip = False
426 filter_in
= yaml
.safe_load(kitem
)
429 "Wrong format at '{}'. Expecting '$<yaml-format>'".format(
433 if isinstance(filter_in
, dict):
434 for index
, item
in enumerate(array_to_change
):
435 for filter_k
, filter_v
in filter_in
.items():
437 not isinstance(item
, dict)
438 or filter_k
not in item
439 or item
[filter_k
] != filter_v
444 # match_found_skip = True
448 indexes
.append(index
)
452 while True: # if not match a ValueError exception will be raise
453 index
= array_to_change
.index(filter_in
, index
)
455 # match_found_skip = True
458 indexes
.append(index
)
463 # if match_found_skip:
467 "Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
471 for index
in indexes
:
474 index
in to_insert_at_index
475 and to_insert_at_index
[index
] != _dict_reference
[k
]
477 # Several different insertions on the same item of the array
479 "Conflict at '{}'. Several insertions on same array index {}".format(
480 ":".join(_key_list
), index
483 to_insert_at_index
[index
] = _dict_reference
[k
]
486 index
in indexes_to_edit_delete
487 and values_to_edit_delete
[index
] != _dict_reference
[k
]
489 # Several different editions on the same item of the array
491 "Conflict at '{}'. Several editions on array index {}".format(
492 ":".join(_key_list
), index
495 indexes_to_edit_delete
.append(index
)
496 values_to_edit_delete
[index
] = _dict_reference
[k
]
499 to_append
[k
] = _dict_reference
[k
]
500 # elif _dict_reference[k] is not None:
501 # raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
502 # ":".join(_key_list)))
504 # edition/deletion is done before insertion
505 indexes_to_edit_delete
.sort(reverse
=True)
506 for index
in indexes_to_edit_delete
:
507 _key_list
[-1] = str(index
)
509 if values_to_edit_delete
[index
] is None: # None->Anything
511 del array_to_change
[index
]
513 pass # it is not consider an error if this index does not exist
515 values_to_edit_delete
[index
], dict
516 ): # NotDict->Anything
517 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
518 elif isinstance(array_to_change
[index
], dict): # Dict->Dict
520 array_to_change
[index
], values_to_edit_delete
[index
], _key_list
522 else: # Dict->NotDict
524 array_to_change
[index
], list
525 ): # Dict->List. Check extra array edition
526 if _deep_update_array(
527 array_to_change
[index
],
528 values_to_edit_delete
[index
],
532 array_to_change
[index
] = deepcopy(values_to_edit_delete
[index
])
533 # calling deep_update_rfc7396 to delete the None values
535 array_to_change
[index
], values_to_edit_delete
[index
], _key_list
539 "Array edition index out of range at '{}'".format(
544 # insertion with indexes
545 to_insert_indexes
= list(to_insert_at_index
.keys())
546 to_insert_indexes
.sort(reverse
=True)
547 for index
in to_insert_indexes
:
548 array_to_change
.insert(index
, to_insert_at_index
[index
])
551 for k
, insert_value
in to_append
.items():
552 _key_list
[-1] = str(k
)
553 insert_value_copy
= deepcopy(insert_value
)
554 if isinstance(insert_value_copy
, dict):
555 # calling deep_update_rfc7396 to delete the None values
556 deep_update_rfc7396(insert_value_copy
, insert_value
, _key_list
)
557 array_to_change
.append(insert_value_copy
)
567 for k
in dict_reference
:
568 key_list
[-1] = str(k
)
569 if dict_reference
[k
] is None: # None->Anything
570 if k
in dict_to_change
:
571 del dict_to_change
[k
]
572 elif not isinstance(dict_reference
[k
], dict): # NotDict->Anything
573 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
574 elif k
not in dict_to_change
: # Dict->Empty
575 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
576 # calling deep_update_rfc7396 to delete the None values
577 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
578 elif isinstance(dict_to_change
[k
], dict): # Dict->Dict
579 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
580 else: # Dict->NotDict
582 dict_to_change
[k
], list
583 ): # Dict->List. Check extra array edition
584 if _deep_update_array(dict_to_change
[k
], dict_reference
[k
], key_list
):
586 dict_to_change
[k
] = deepcopy(dict_reference
[k
])
587 # calling deep_update_rfc7396 to delete the None values
588 deep_update_rfc7396(dict_to_change
[k
], dict_reference
[k
], key_list
)
592 def deep_update(dict_to_change
, dict_reference
):
593 """Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
594 return deep_update_rfc7396(dict_to_change
, dict_reference
)