osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22 from Crypto.Cipher import AES
  23 from base64 import b64decode, b64encode
  24
  25 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  26
  27
  28 class DbException(Exception):
  29
  30     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  31         self.http_code = http_code
  32         Exception.__init__(self, "database exception " + str(message))
  33
  34
  35 class DbBase(object):
  36
  37     def __init__(self, logger_name='db', master_password=None):
  38         """
  39         Constructor od dbBase
  40         :param logger_name: logging name
  41         :param master_password: master password used for encrypt decrypt methods
  42         """
  43         self.logger = logging.getLogger(logger_name)
  44         self.master_password = master_password
  45         self.secret_key = None
  46
  47     def db_connect(self, config, target_version=None):
  48         """
  49         Connect to database
  50         :param config: Configuration of database
  51         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  52         :return: None or raises DbException on error
  53         """
  54         raise DbException("Method 'db_connect' not implemented")
  55
  56     def db_disconnect(self):
  57         """
  58         Disconnect from database
  59         :return: None
  60         """
  61         pass
  62
  63     def get_list(self, table, q_filter=None):
  64         """
  65         Obtain a list of entries matching q_filter
  66         :param table: collection or table
  67         :param q_filter: Filter
  68         :return: a list (can be empty) with the found entries. Raises DbException on error
  69         """
  70         raise DbException("Method 'get_list' not implemented")
  71
  72     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
  73         """
  74         Obtain one entry matching q_filter
  75         :param table: collection or table
  76         :param q_filter: Filter
  77         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
  78         it raises a DbException
  79         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
  80         that it raises a DbException
  81         :return: The requested element, or None
  82         """
  83         raise DbException("Method 'get_one' not implemented")
  84
  85     def del_list(self, table, q_filter=None):
  86         """
  87         Deletes all entries that match q_filter
  88         :param table: collection or table
  89         :param q_filter: Filter
  90         :return: Dict with the number of entries deleted
  91         """
  92         raise DbException("Method 'del_list' not implemented")
  93
  94     def del_one(self, table, q_filter=None, fail_on_empty=True):
  95         """
  96         Deletes one entry that matches q_filter
  97         :param table: collection or table
  98         :param q_filter: Filter
  99         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 100         which case it raises a DbException
 101         :return: Dict with the number of entries deleted
 102         """
 103         raise DbException("Method 'del_one' not implemented")
 104
 105     def create(self, table, indata):
 106         """
 107         Add a new entry at database
 108         :param table: collection or table
 109         :param indata: content to be added
 110         :return: database id of the inserted element. Raises a DbException on error
 111         """
 112         raise DbException("Method 'create' not implemented")
 113
 114     def set_one(self, table, q_filter, update_dict, fail_on_empty=True):
 115         """
 116         Modifies an entry at database
 117         :param table: collection or table
 118         :param q_filter: Filter
 119         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 120         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 121         it raises a DbException
 122         :return: Dict with the number of entries modified. None if no matching is found.
 123         """
 124         raise DbException("Method 'set_one' not implemented")
 125
 126     def set_list(self, table, q_filter, update_dict):
 127         """
 128         Modifies al matching entries at database
 129         :param table: collection or table
 130         :param q_filter: Filter
 131         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 132         :return: Dict with the number of entries modified
 133         """
 134         raise DbException("Method 'set_list' not implemented")
 135
 136     def replace(self, table, _id, indata, fail_on_empty=True):
 137         """
 138         Replace the content of an entry
 139         :param table: collection or table
 140         :param _id: internal database id
 141         :param indata: content to replace
 142         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 143         it raises a DbException
 144         :return: Dict with the number of entries replaced
 145         """
 146         raise DbException("Method 'replace' not implemented")
 147
 148     @staticmethod
 149     def _join_passwords(passwd_byte, passwd_str):
 150         """
 151         Modifies passwd_byte with the xor of passwd_str. Used for adding salt, join passwords, etc
 152         :param passwd_byte: original password in bytes, 32 byte length
 153         :param passwd_str: string salt to be added
 154         :return: modified password in bytes
 155         """
 156         if not passwd_str:
 157             return passwd_byte
 158         secret_key = bytearray(passwd_byte)
 159         for i, b in enumerate(passwd_str.encode()):
 160             secret_key[i % 32] ^= b
 161         return bytes(secret_key)
 162
 163     def set_secret_key(self, secret_key):
 164         """
 165         Set internal secret key used for encryption
 166         :param secret_key: byte array length 32 with the secret_key
 167         :return: None
 168         """
 169         assert (len(secret_key) == 32)
 170         self.secret_key = self._join_passwords(secret_key, self.master_password)
 171
 172     def encrypt(self, value, schema_version=None, salt=None):
 173         """
 174         Encrypt a value
 175         :param value: value to be encrypted. It is string/unicode
 176         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 177                If '1.1' symmetric AES encryption is done
 178         :param salt: optional salt to be used. Must be str
 179         :return: Encrypted content of value
 180         """
 181         if not schema_version or schema_version == '1.0':
 182             return value
 183         else:
 184             if not self.secret_key:
 185                 raise DbException("Cannot encrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 186             secret_key = self._join_passwords(self.secret_key, salt)
 187             cipher = AES.new(secret_key)
 188             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 189             encrypted_msg = cipher.encrypt(padded_private_msg)
 190             encoded_encrypted_msg = b64encode(encrypted_msg)
 191             return encoded_encrypted_msg.decode("ascii")
 192
 193     def decrypt(self, value, schema_version=None, salt=None):
 194         """
 195         Decrypt an encrypted value
 196         :param value: value to be decrypted. It is a base64 string
 197         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 198                If '1.1' symmetric AES encryption has been done
 199         :param salt: optional salt to be used
 200         :return: Plain content of value
 201         """
 202         if not schema_version or schema_version == '1.0':
 203             return value
 204         else:
 205             if not self.secret_key:
 206                 raise DbException("Cannot decrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 207             secret_key = self._join_passwords(self.secret_key, salt)
 208             encrypted_msg = b64decode(value)
 209             cipher = AES.new(secret_key)
 210             decrypted_msg = cipher.decrypt(encrypted_msg)
 211             unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 212             return unpadded_private_msg
 213
 214
 215 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 216     """
 217     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 218     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 219     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 220     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 221         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 222                     the element of the array is deleted, otherwise it is edited.
 223         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 224         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 225         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 226                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 227                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 228         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 229                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 230                     format '{key: val, key: val, ...}'; and all of them mast match. Nothing happens if no match is
 231                     found. If value is None the matched items are deleted, otherwise they are edited.
 232         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 233                     is changed. A value of None has not sense.
 234         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 235                     nothing is changed. A value of None has not sense.
 236     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 237         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 238         insertions must take into account the deleted items.
 239     :param dict_to_change:  Target dictionary to be changed.
 240     :param dict_reference: Dictionary that contains changes to be applied.
 241     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 242     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 243     """
 244     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 245         to_append = {}
 246         to_insert_at_index = {}
 247         values_to_edit_delete = {}
 248         indexes_to_edit_delete = []
 249         array_edition = None
 250         _key_list.append("")
 251         for k in _dict_reference:
 252             _key_list[-1] = str(k)
 253             if not isinstance(k, str) or not k.startswith("$"):
 254                 if array_edition is True:
 255                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 256                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 257                 array_edition = False
 258                 continue
 259             else:
 260                 if array_edition is False:
 261                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 262                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 263                 array_edition = True
 264             insert = False
 265             indexes = []  # indexes to edit or insert
 266             kitem = k[1:]
 267             if kitem.startswith('+'):
 268                 insert = True
 269                 kitem = kitem[1:]
 270                 if _dict_reference[k] is None:
 271                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 272                         ":".join(_key_list)))
 273
 274             if kitem.startswith('[') and kitem.endswith(']'):
 275                 try:
 276                     index = int(kitem[1:-1])
 277                     if index < 0:
 278                         index += len(array_to_change)
 279                     if index < 0:
 280                         index = 0  # skip outside index edition
 281                     indexes.append(index)
 282                 except Exception:
 283                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 284                         ":".join(_key_list)))
 285             elif kitem:
 286                 # match_found_skip = False
 287                 try:
 288                     filter_in = yaml.safe_load(kitem)
 289                 except Exception:
 290                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 291                 if isinstance(filter_in, dict):
 292                     for index, item in enumerate(array_to_change):
 293                         for filter_k, filter_v in filter_in.items():
 294                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 295                                 break
 296                         else:  # match found
 297                             if insert:
 298                                 # match_found_skip = True
 299                                 insert = False
 300                                 break
 301                             else:
 302                                 indexes.append(index)
 303                 else:
 304                     index = 0
 305                     try:
 306                         while True:  # if not match a ValueError exception will be raise
 307                             index = array_to_change.index(filter_in, index)
 308                             if insert:
 309                                 # match_found_skip = True
 310                                 insert = False
 311                                 break
 312                             indexes.append(index)
 313                             index += 1
 314                     except ValueError:
 315                         pass
 316
 317                 # if match_found_skip:
 318                 #     continue
 319             elif not insert:
 320                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 321                     ":".join(_key_list)))
 322             for index in indexes:
 323                 if insert:
 324                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 325                         # Several different insertions on the same item of the array
 326                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 327                             ":".join(_key_list), index))
 328                     to_insert_at_index[index] = _dict_reference[k]
 329                 else:
 330                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 331                         # Several different editions on the same item of the array
 332                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 333                             ":".join(_key_list), index))
 334                     indexes_to_edit_delete.append(index)
 335                     values_to_edit_delete[index] = _dict_reference[k]
 336             if not indexes:
 337                 if insert:
 338                     to_append[k] = _dict_reference[k]
 339                 # elif _dict_reference[k] is not None:
 340                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 341                 #         ":".join(_key_list)))
 342
 343         # edition/deletion is done before insertion
 344         indexes_to_edit_delete.sort(reverse=True)
 345         for index in indexes_to_edit_delete:
 346             _key_list[-1] = str(index)
 347             try:
 348                 if values_to_edit_delete[index] is None:  # None->Anything
 349                     try:
 350                         del (array_to_change[index])
 351                     except IndexError:
 352                         pass  # it is not consider an error if this index does not exist
 353                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 354                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 355                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 356                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 357                 else:  # Dict->NotDict
 358                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 359                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 360                             continue
 361                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 362                     # calling deep_update_rfc7396 to delete the None values
 363                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 364             except IndexError:
 365                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 366
 367         # insertion with indexes
 368         to_insert_indexes = list(to_insert_at_index.keys())
 369         to_insert_indexes.sort(reverse=True)
 370         for index in to_insert_indexes:
 371             array_to_change.insert(index, to_insert_at_index[index])
 372
 373         # append
 374         for k, insert_value in to_append.items():
 375             _key_list[-1] = str(k)
 376             insert_value_copy = deepcopy(insert_value)
 377             if isinstance(insert_value_copy, dict):
 378                 # calling deep_update_rfc7396 to delete the None values
 379                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 380             array_to_change.append(insert_value_copy)
 381
 382         _key_list.pop()
 383         if array_edition:
 384             return True
 385         return False
 386
 387     if key_list is None:
 388         key_list = []
 389     key_list.append("")
 390     for k in dict_reference:
 391         key_list[-1] = str(k)
 392         if dict_reference[k] is None:   # None->Anything
 393             if k in dict_to_change:
 394                 del dict_to_change[k]
 395         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 396             dict_to_change[k] = deepcopy(dict_reference[k])
 397         elif k not in dict_to_change:  # Dict->Empty
 398             dict_to_change[k] = deepcopy(dict_reference[k])
 399             # calling deep_update_rfc7396 to delete the None values
 400             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 401         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 402             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 403         else:       # Dict->NotDict
 404             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 405                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 406                     continue
 407             dict_to_change[k] = deepcopy(dict_reference[k])
 408             # calling deep_update_rfc7396 to delete the None values
 409             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 410     key_list.pop()
 411
 412
 413 def deep_update(dict_to_change, dict_reference):
 414     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 415     return deep_update_rfc7396(dict_to_change, dict_reference)