osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22 from Crypto.Cipher import AES
  23 from base64 import b64decode, b64encode
  24 from osm_common.common_utils import FakeLock
  25 from threading import Lock
  26
  27 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  28
  29
  30 class DbException(Exception):
  31
  32     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  33         self.http_code = http_code
  34         Exception.__init__(self, "database exception " + str(message))
  35
  36
  37 class DbBase(object):
  38
  39     def __init__(self, logger_name='db', lock=False):
  40         """
  41         Constructor of dbBase
  42         :param logger_name: logging name
  43         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  44             False, None: Do not protect, this object will only be accessed by one thread
  45             True: This object needs to be protected by several threads accessing.
  46             Lock object. Use thi Lock for the threads access protection
  47         """
  48         self.logger = logging.getLogger(logger_name)
  49         self.master_password = None
  50         self.secret_key = None
  51         if not lock:
  52             self.lock = FakeLock()
  53         elif lock is True:
  54             self.lock = Lock()
  55         elif isinstance(lock, Lock):
  56             self.lock = lock
  57         else:
  58             raise ValueError("lock parameter must be a Lock classclass or boolean")
  59
  60     def db_connect(self, config, target_version=None):
  61         """
  62         Connect to database
  63         :param config: Configuration of database. Contains among others:
  64             host:   database hosst (mandatory)
  65             port:   database port (mandatory)
  66             name:   database name (mandatory)
  67             user:   database username
  68             password:   database password
  69             masterpassword: database password used for sensible information encryption
  70         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  71         :return: None or raises DbException on error
  72         """
  73         raise DbException("Method 'db_connect' not implemented")
  74
  75     def db_disconnect(self):
  76         """
  77         Disconnect from database
  78         :return: None
  79         """
  80         pass
  81
  82     def get_list(self, table, q_filter=None):
  83         """
  84         Obtain a list of entries matching q_filter
  85         :param table: collection or table
  86         :param q_filter: Filter
  87         :return: a list (can be empty) with the found entries. Raises DbException on error
  88         """
  89         raise DbException("Method 'get_list' not implemented")
  90
  91     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
  92         """
  93         Obtain one entry matching q_filter
  94         :param table: collection or table
  95         :param q_filter: Filter
  96         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
  97         it raises a DbException
  98         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
  99         that it raises a DbException
 100         :return: The requested element, or None
 101         """
 102         raise DbException("Method 'get_one' not implemented")
 103
 104     def del_list(self, table, q_filter=None):
 105         """
 106         Deletes all entries that match q_filter
 107         :param table: collection or table
 108         :param q_filter: Filter
 109         :return: Dict with the number of entries deleted
 110         """
 111         raise DbException("Method 'del_list' not implemented")
 112
 113     def del_one(self, table, q_filter=None, fail_on_empty=True):
 114         """
 115         Deletes one entry that matches q_filter
 116         :param table: collection or table
 117         :param q_filter: Filter
 118         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 119         which case it raises a DbException
 120         :return: Dict with the number of entries deleted
 121         """
 122         raise DbException("Method 'del_one' not implemented")
 123
 124     def create(self, table, indata):
 125         """
 126         Add a new entry at database
 127         :param table: collection or table
 128         :param indata: content to be added
 129         :return: database id of the inserted element. Raises a DbException on error
 130         """
 131         raise DbException("Method 'create' not implemented")
 132
 133     def set_one(self, table, q_filter, update_dict, fail_on_empty=True):
 134         """
 135         Modifies an entry at database
 136         :param table: collection or table
 137         :param q_filter: Filter
 138         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 139         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 140         it raises a DbException
 141         :return: Dict with the number of entries modified. None if no matching is found.
 142         """
 143         raise DbException("Method 'set_one' not implemented")
 144
 145     def set_list(self, table, q_filter, update_dict):
 146         """
 147         Modifies al matching entries at database
 148         :param table: collection or table
 149         :param q_filter: Filter
 150         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 151         :return: Dict with the number of entries modified
 152         """
 153         raise DbException("Method 'set_list' not implemented")
 154
 155     def replace(self, table, _id, indata, fail_on_empty=True):
 156         """
 157         Replace the content of an entry
 158         :param table: collection or table
 159         :param _id: internal database id
 160         :param indata: content to replace
 161         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 162         it raises a DbException
 163         :return: Dict with the number of entries replaced
 164         """
 165         raise DbException("Method 'replace' not implemented")
 166
 167     @staticmethod
 168     def _join_passwords(passwd_byte, passwd_str):
 169         """
 170         Modifies passwd_byte with the xor of passwd_str. Used for adding salt, join passwords, etc
 171         :param passwd_byte: original password in bytes, 32 byte length
 172         :param passwd_str: string salt to be added
 173         :return: modified password in bytes
 174         """
 175         if not passwd_str:
 176             return passwd_byte
 177         secret_key = bytearray(passwd_byte)
 178         for i, b in enumerate(passwd_str.encode()):
 179             secret_key[i % 32] ^= b
 180         return bytes(secret_key)
 181
 182     def set_secret_key(self, secret_key):
 183         """
 184         Set internal secret key used for encryption
 185         :param secret_key: byte array length 32 with the secret_key
 186         :return: None
 187         """
 188         assert (len(secret_key) == 32)
 189         self.secret_key = self._join_passwords(secret_key, self.master_password)
 190
 191     def encrypt(self, value, schema_version=None, salt=None):
 192         """
 193         Encrypt a value
 194         :param value: value to be encrypted. It is string/unicode
 195         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 196                If '1.1' symmetric AES encryption is done
 197         :param salt: optional salt to be used. Must be str
 198         :return: Encrypted content of value
 199         """
 200         if not schema_version or schema_version == '1.0':
 201             return value
 202         else:
 203             if not self.secret_key:
 204                 raise DbException("Cannot encrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 205             secret_key = self._join_passwords(self.secret_key, salt)
 206             cipher = AES.new(secret_key)
 207             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 208             encrypted_msg = cipher.encrypt(padded_private_msg)
 209             encoded_encrypted_msg = b64encode(encrypted_msg)
 210             return encoded_encrypted_msg.decode("ascii")
 211
 212     def decrypt(self, value, schema_version=None, salt=None):
 213         """
 214         Decrypt an encrypted value
 215         :param value: value to be decrypted. It is a base64 string
 216         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 217                If '1.1' symmetric AES encryption has been done
 218         :param salt: optional salt to be used
 219         :return: Plain content of value
 220         """
 221         if not schema_version or schema_version == '1.0':
 222             return value
 223         else:
 224             if not self.secret_key:
 225                 raise DbException("Cannot decrypt. Missing secret_key", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 226             secret_key = self._join_passwords(self.secret_key, salt)
 227             encrypted_msg = b64decode(value)
 228             cipher = AES.new(secret_key)
 229             decrypted_msg = cipher.decrypt(encrypted_msg)
 230             unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 231             return unpadded_private_msg
 232
 233
 234 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 235     """
 236     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 237     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 238     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 239     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 240         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 241                     the element of the array is deleted, otherwise it is edited.
 242         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 243         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 244         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 245                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 246                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 247         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 248                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 249                     format '{key: val, key: val, ...}'; and all of them mast match. Nothing happens if no match is
 250                     found. If value is None the matched items are deleted, otherwise they are edited.
 251         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 252                     is changed. A value of None has not sense.
 253         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 254                     nothing is changed. A value of None has not sense.
 255     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 256         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 257         insertions must take into account the deleted items.
 258     :param dict_to_change:  Target dictionary to be changed.
 259     :param dict_reference: Dictionary that contains changes to be applied.
 260     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 261     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 262     """
 263     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 264         to_append = {}
 265         to_insert_at_index = {}
 266         values_to_edit_delete = {}
 267         indexes_to_edit_delete = []
 268         array_edition = None
 269         _key_list.append("")
 270         for k in _dict_reference:
 271             _key_list[-1] = str(k)
 272             if not isinstance(k, str) or not k.startswith("$"):
 273                 if array_edition is True:
 274                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 275                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 276                 array_edition = False
 277                 continue
 278             else:
 279                 if array_edition is False:
 280                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 281                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 282                 array_edition = True
 283             insert = False
 284             indexes = []  # indexes to edit or insert
 285             kitem = k[1:]
 286             if kitem.startswith('+'):
 287                 insert = True
 288                 kitem = kitem[1:]
 289                 if _dict_reference[k] is None:
 290                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 291                         ":".join(_key_list)))
 292
 293             if kitem.startswith('[') and kitem.endswith(']'):
 294                 try:
 295                     index = int(kitem[1:-1])
 296                     if index < 0:
 297                         index += len(array_to_change)
 298                     if index < 0:
 299                         index = 0  # skip outside index edition
 300                     indexes.append(index)
 301                 except Exception:
 302                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 303                         ":".join(_key_list)))
 304             elif kitem:
 305                 # match_found_skip = False
 306                 try:
 307                     filter_in = yaml.safe_load(kitem)
 308                 except Exception:
 309                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 310                 if isinstance(filter_in, dict):
 311                     for index, item in enumerate(array_to_change):
 312                         for filter_k, filter_v in filter_in.items():
 313                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 314                                 break
 315                         else:  # match found
 316                             if insert:
 317                                 # match_found_skip = True
 318                                 insert = False
 319                                 break
 320                             else:
 321                                 indexes.append(index)
 322                 else:
 323                     index = 0
 324                     try:
 325                         while True:  # if not match a ValueError exception will be raise
 326                             index = array_to_change.index(filter_in, index)
 327                             if insert:
 328                                 # match_found_skip = True
 329                                 insert = False
 330                                 break
 331                             indexes.append(index)
 332                             index += 1
 333                     except ValueError:
 334                         pass
 335
 336                 # if match_found_skip:
 337                 #     continue
 338             elif not insert:
 339                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 340                     ":".join(_key_list)))
 341             for index in indexes:
 342                 if insert:
 343                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 344                         # Several different insertions on the same item of the array
 345                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 346                             ":".join(_key_list), index))
 347                     to_insert_at_index[index] = _dict_reference[k]
 348                 else:
 349                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 350                         # Several different editions on the same item of the array
 351                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 352                             ":".join(_key_list), index))
 353                     indexes_to_edit_delete.append(index)
 354                     values_to_edit_delete[index] = _dict_reference[k]
 355             if not indexes:
 356                 if insert:
 357                     to_append[k] = _dict_reference[k]
 358                 # elif _dict_reference[k] is not None:
 359                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 360                 #         ":".join(_key_list)))
 361
 362         # edition/deletion is done before insertion
 363         indexes_to_edit_delete.sort(reverse=True)
 364         for index in indexes_to_edit_delete:
 365             _key_list[-1] = str(index)
 366             try:
 367                 if values_to_edit_delete[index] is None:  # None->Anything
 368                     try:
 369                         del (array_to_change[index])
 370                     except IndexError:
 371                         pass  # it is not consider an error if this index does not exist
 372                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 373                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 374                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 375                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 376                 else:  # Dict->NotDict
 377                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 378                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 379                             continue
 380                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 381                     # calling deep_update_rfc7396 to delete the None values
 382                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 383             except IndexError:
 384                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 385
 386         # insertion with indexes
 387         to_insert_indexes = list(to_insert_at_index.keys())
 388         to_insert_indexes.sort(reverse=True)
 389         for index in to_insert_indexes:
 390             array_to_change.insert(index, to_insert_at_index[index])
 391
 392         # append
 393         for k, insert_value in to_append.items():
 394             _key_list[-1] = str(k)
 395             insert_value_copy = deepcopy(insert_value)
 396             if isinstance(insert_value_copy, dict):
 397                 # calling deep_update_rfc7396 to delete the None values
 398                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 399             array_to_change.append(insert_value_copy)
 400
 401         _key_list.pop()
 402         if array_edition:
 403             return True
 404         return False
 405
 406     if key_list is None:
 407         key_list = []
 408     key_list.append("")
 409     for k in dict_reference:
 410         key_list[-1] = str(k)
 411         if dict_reference[k] is None:   # None->Anything
 412             if k in dict_to_change:
 413                 del dict_to_change[k]
 414         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 415             dict_to_change[k] = deepcopy(dict_reference[k])
 416         elif k not in dict_to_change:  # Dict->Empty
 417             dict_to_change[k] = deepcopy(dict_reference[k])
 418             # calling deep_update_rfc7396 to delete the None values
 419             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 420         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 421             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 422         else:       # Dict->NotDict
 423             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 424                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 425                     continue
 426             dict_to_change[k] = deepcopy(dict_reference[k])
 427             # calling deep_update_rfc7396 to delete the None values
 428             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 429     key_list.pop()
 430
 431
 432 def deep_update(dict_to_change, dict_reference):
 433     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 434     return deep_update_rfc7396(dict_to_change, dict_reference)