osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22 from Crypto.Cipher import AES
  23 from base64 import b64decode, b64encode
  24
  25 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  26
  27
  28 class DbException(Exception):
  29
  30     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  31         self.http_code = http_code
  32         Exception.__init__(self, "database exception " + str(message))
  33
  34
  35 class DbBase(object):
  36
  37     def __init__(self, logger_name='db'):
  38         """
  39         Constructor od dbBase
  40         :param logger_name: logging name
  41         """
  42         self.logger = logging.getLogger(logger_name)
  43         self.master_password = None
  44         self.secret_key = None
  45
  46     def db_connect(self, config, target_version=None):
  47         """
  48         Connect to database
  49         :param config: Configuration of database. Contains among others:
  50             host:   database hosst (mandatory)
  51             port:   database port (mandatory)
  52             name:   database name (mandatory)
  53             user:   database username
  54             password:   database password
  55             masterpassword: database password used for sensible information encryption
  56         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  57         :return: None or raises DbException on error
  58         """
  59         raise DbException("Method 'db_connect' not implemented")
  60
  61     def db_disconnect(self):
  62         """
  63         Disconnect from database
  64         :return: None
  65         """
  66         pass
  67
  68     def get_list(self, table, q_filter=None):
  69         """
  70         Obtain a list of entries matching q_filter
  71         :param table: collection or table
  72         :param q_filter: Filter
  73         :return: a list (can be empty) with the found entries. Raises DbException on error
  74         """
  75         raise DbException("Method 'get_list' not implemented")
  76
  77     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
  78         """
  79         Obtain one entry matching q_filter
  80         :param table: collection or table
  81         :param q_filter: Filter
  82         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
  83         it raises a DbException
  84         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
  85         that it raises a DbException
  86         :return: The requested element, or None
  87         """
  88         raise DbException("Method 'get_one' not implemented")
  89
  90     def del_list(self, table, q_filter=None):
  91         """
  92         Deletes all entries that match q_filter
  93         :param table: collection or table
  94         :param q_filter: Filter
  95         :return: Dict with the number of entries deleted
  96         """
  97         raise DbException("Method 'del_list' not implemented")
  98
  99     def del_one(self, table, q_filter=None, fail_on_empty=True):
 100         """
 101         Deletes one entry that matches q_filter
 102         :param table: collection or table
 103         :param q_filter: Filter
 104         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 105         which case it raises a DbException
 106         :return: Dict with the number of entries deleted
 107         """
 108         raise DbException("Method 'del_one' not implemented")
 109
 110     def create(self, table, indata):
 111         """
 112         Add a new entry at database
 113         :param table: collection or table
 114         :param indata: content to be added
 115         :return: database id of the inserted element. Raises a DbException on error
 116         """
 117         raise DbException("Method 'create' not implemented")
 118
 119     def set_one(self, table, q_filter, update_dict, fail_on_empty=True):
 120         """
 121         Modifies an entry at database
 122         :param table: collection or table
 123         :param q_filter: Filter
 124         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 125         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 126         it raises a DbException
 127         :return: Dict with the number of entries modified. None if no matching is found.
 128         """
 129         raise DbException("Method 'set_one' not implemented")
 130
 131     def set_list(self, table, q_filter, update_dict):
 132         """
 133         Modifies al matching entries at database
 134         :param table: collection or table
 135         :param q_filter: Filter
 136         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 137         :return: Dict with the number of entries modified
 138         """
 139         raise DbException("Method 'set_list' not implemented")
 140
 141     def replace(self, table, _id, indata, fail_on_empty=True):
 142         """
 143         Replace the content of an entry
 144         :param table: collection or table
 145         :param _id: internal database id
 146         :param indata: content to replace
 147         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 148         it raises a DbException
 149         :return: Dict with the number of entries replaced
 150         """
 151         raise DbException("Method 'replace' not implemented")
 152
 153     @staticmethod
 154     def _join_passwords(passwd_byte, passwd_str):
 155         """
 156         Modifies passwd_byte with the xor of passwd_str. Used for adding salt, join passwords, etc
 157         :param passwd_byte: original password in bytes, 32 byte length
 158         :param passwd_str: string salt to be added
 159         :return: modified password in bytes
 160         """
 161         if not passwd_str:
 162             return passwd_byte
 163         secret_key = bytearray(passwd_byte)
 164         for i, b in enumerate(passwd_str.encode()):
 165             secret_key[i % 32] ^= b
 166         return bytes(secret_key)
 167
 168     def set_secret_key(self, secret_key):
 169         """
 170         Set internal secret key used for encryption
 171         :param secret_key: byte array length 32 with the secret_key
 172         :return: None
 173         """
 174         assert (len(secret_key) == 32)
 175         self.secret_key = self._join_passwords(secret_key, self.master_password)
 176
 177     def encrypt(self, value, schema_version=None, salt=None):
 178         """
 179         Encrypt a value
 180         :param value: value to be encrypted. It is string/unicode
 181         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 182                If '1.1' symmetric AES encryption is done
 183         :param salt: optional salt to be used. Must be str
 184         :return: Encrypted content of value
 185         """
 186         if not schema_version or schema_version == '1.0':
 187             return value
 188         else:
 189             if not self.secret_key:
 190                 raise DbException("Cannot encrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 191             secret_key = self._join_passwords(self.secret_key, salt)
 192             cipher = AES.new(secret_key)
 193             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 194             encrypted_msg = cipher.encrypt(padded_private_msg)
 195             encoded_encrypted_msg = b64encode(encrypted_msg)
 196             return encoded_encrypted_msg.decode("ascii")
 197
 198     def decrypt(self, value, schema_version=None, salt=None):
 199         """
 200         Decrypt an encrypted value
 201         :param value: value to be decrypted. It is a base64 string
 202         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 203                If '1.1' symmetric AES encryption has been done
 204         :param salt: optional salt to be used
 205         :return: Plain content of value
 206         """
 207         if not schema_version or schema_version == '1.0':
 208             return value
 209         else:
 210             if not self.secret_key:
 211                 raise DbException("Cannot decrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 212             secret_key = self._join_passwords(self.secret_key, salt)
 213             encrypted_msg = b64decode(value)
 214             cipher = AES.new(secret_key)
 215             decrypted_msg = cipher.decrypt(encrypted_msg)
 216             unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 217             return unpadded_private_msg
 218
 219
 220 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 221     """
 222     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 223     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 224     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 225     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 226         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 227                     the element of the array is deleted, otherwise it is edited.
 228         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 229         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 230         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 231                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 232                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 233         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 234                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 235                     format '{key: val, key: val, ...}'; and all of them mast match. Nothing happens if no match is
 236                     found. If value is None the matched items are deleted, otherwise they are edited.
 237         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 238                     is changed. A value of None has not sense.
 239         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 240                     nothing is changed. A value of None has not sense.
 241     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 242         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 243         insertions must take into account the deleted items.
 244     :param dict_to_change:  Target dictionary to be changed.
 245     :param dict_reference: Dictionary that contains changes to be applied.
 246     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 247     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 248     """
 249     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 250         to_append = {}
 251         to_insert_at_index = {}
 252         values_to_edit_delete = {}
 253         indexes_to_edit_delete = []
 254         array_edition = None
 255         _key_list.append("")
 256         for k in _dict_reference:
 257             _key_list[-1] = str(k)
 258             if not isinstance(k, str) or not k.startswith("$"):
 259                 if array_edition is True:
 260                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 261                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 262                 array_edition = False
 263                 continue
 264             else:
 265                 if array_edition is False:
 266                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 267                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 268                 array_edition = True
 269             insert = False
 270             indexes = []  # indexes to edit or insert
 271             kitem = k[1:]
 272             if kitem.startswith('+'):
 273                 insert = True
 274                 kitem = kitem[1:]
 275                 if _dict_reference[k] is None:
 276                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 277                         ":".join(_key_list)))
 278
 279             if kitem.startswith('[') and kitem.endswith(']'):
 280                 try:
 281                     index = int(kitem[1:-1])
 282                     if index < 0:
 283                         index += len(array_to_change)
 284                     if index < 0:
 285                         index = 0  # skip outside index edition
 286                     indexes.append(index)
 287                 except Exception:
 288                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 289                         ":".join(_key_list)))
 290             elif kitem:
 291                 # match_found_skip = False
 292                 try:
 293                     filter_in = yaml.safe_load(kitem)
 294                 except Exception:
 295                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 296                 if isinstance(filter_in, dict):
 297                     for index, item in enumerate(array_to_change):
 298                         for filter_k, filter_v in filter_in.items():
 299                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 300                                 break
 301                         else:  # match found
 302                             if insert:
 303                                 # match_found_skip = True
 304                                 insert = False
 305                                 break
 306                             else:
 307                                 indexes.append(index)
 308                 else:
 309                     index = 0
 310                     try:
 311                         while True:  # if not match a ValueError exception will be raise
 312                             index = array_to_change.index(filter_in, index)
 313                             if insert:
 314                                 # match_found_skip = True
 315                                 insert = False
 316                                 break
 317                             indexes.append(index)
 318                             index += 1
 319                     except ValueError:
 320                         pass
 321
 322                 # if match_found_skip:
 323                 #     continue
 324             elif not insert:
 325                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 326                     ":".join(_key_list)))
 327             for index in indexes:
 328                 if insert:
 329                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 330                         # Several different insertions on the same item of the array
 331                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 332                             ":".join(_key_list), index))
 333                     to_insert_at_index[index] = _dict_reference[k]
 334                 else:
 335                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 336                         # Several different editions on the same item of the array
 337                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 338                             ":".join(_key_list), index))
 339                     indexes_to_edit_delete.append(index)
 340                     values_to_edit_delete[index] = _dict_reference[k]
 341             if not indexes:
 342                 if insert:
 343                     to_append[k] = _dict_reference[k]
 344                 # elif _dict_reference[k] is not None:
 345                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 346                 #         ":".join(_key_list)))
 347
 348         # edition/deletion is done before insertion
 349         indexes_to_edit_delete.sort(reverse=True)
 350         for index in indexes_to_edit_delete:
 351             _key_list[-1] = str(index)
 352             try:
 353                 if values_to_edit_delete[index] is None:  # None->Anything
 354                     try:
 355                         del (array_to_change[index])
 356                     except IndexError:
 357                         pass  # it is not consider an error if this index does not exist
 358                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 359                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 360                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 361                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 362                 else:  # Dict->NotDict
 363                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 364                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 365                             continue
 366                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 367                     # calling deep_update_rfc7396 to delete the None values
 368                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 369             except IndexError:
 370                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 371
 372         # insertion with indexes
 373         to_insert_indexes = list(to_insert_at_index.keys())
 374         to_insert_indexes.sort(reverse=True)
 375         for index in to_insert_indexes:
 376             array_to_change.insert(index, to_insert_at_index[index])
 377
 378         # append
 379         for k, insert_value in to_append.items():
 380             _key_list[-1] = str(k)
 381             insert_value_copy = deepcopy(insert_value)
 382             if isinstance(insert_value_copy, dict):
 383                 # calling deep_update_rfc7396 to delete the None values
 384                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 385             array_to_change.append(insert_value_copy)
 386
 387         _key_list.pop()
 388         if array_edition:
 389             return True
 390         return False
 391
 392     if key_list is None:
 393         key_list = []
 394     key_list.append("")
 395     for k in dict_reference:
 396         key_list[-1] = str(k)
 397         if dict_reference[k] is None:   # None->Anything
 398             if k in dict_to_change:
 399                 del dict_to_change[k]
 400         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 401             dict_to_change[k] = deepcopy(dict_reference[k])
 402         elif k not in dict_to_change:  # Dict->Empty
 403             dict_to_change[k] = deepcopy(dict_reference[k])
 404             # calling deep_update_rfc7396 to delete the None values
 405             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 406         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 407             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 408         else:       # Dict->NotDict
 409             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 410                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 411                     continue
 412             dict_to_change[k] = deepcopy(dict_reference[k])
 413             # calling deep_update_rfc7396 to delete the None values
 414             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 415     key_list.pop()
 416
 417
 418 def deep_update(dict_to_change, dict_reference):
 419     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 420     return deep_update_rfc7396(dict_to_change, dict_reference)