osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22 from Crypto.Cipher import AES
  23 from base64 import b64decode, b64encode
  24 from osm_common.common_utils import FakeLock
  25 from threading import Lock
  26
  27 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  28
  29
  30 class DbException(Exception):
  31
  32     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  33         self.http_code = http_code
  34         Exception.__init__(self, "database exception " + str(message))
  35
  36
  37 class DbBase(object):
  38
  39     def __init__(self, logger_name='db', lock=False):
  40         """
  41         Constructor of dbBase
  42         :param logger_name: logging name
  43         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  44             False, None: Do not protect, this object will only be accessed by one thread
  45             True: This object needs to be protected by several threads accessing.
  46             Lock object. Use thi Lock for the threads access protection
  47         """
  48         self.logger = logging.getLogger(logger_name)
  49         self.secret_key = None  # 32 bytes length array used for encrypt/decrypt
  50         if not lock:
  51             self.lock = FakeLock()
  52         elif lock is True:
  53             self.lock = Lock()
  54         elif isinstance(lock, Lock):
  55             self.lock = lock
  56         else:
  57             raise ValueError("lock parameter must be a Lock classclass or boolean")
  58
  59     def db_connect(self, config, target_version=None):
  60         """
  61         Connect to database
  62         :param config: Configuration of database. Contains among others:
  63             host:   database hosst (mandatory)
  64             port:   database port (mandatory)
  65             name:   database name (mandatory)
  66             user:   database username
  67             password:   database password
  68             commonkey: common OSM key used for sensible information encryption
  69             materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
  70         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  71         :return: None or raises DbException on error
  72         """
  73         raise DbException("Method 'db_connect' not implemented")
  74
  75     def db_disconnect(self):
  76         """
  77         Disconnect from database
  78         :return: None
  79         """
  80         pass
  81
  82     def get_list(self, table, q_filter=None):
  83         """
  84         Obtain a list of entries matching q_filter
  85         :param table: collection or table
  86         :param q_filter: Filter
  87         :return: a list (can be empty) with the found entries. Raises DbException on error
  88         """
  89         raise DbException("Method 'get_list' not implemented")
  90
  91     def count(self, table, q_filter=None):
  92         """
  93         Count the number of entries matching q_filter
  94         :param table: collection or table
  95         :param q_filter: Filter
  96         :return: number of entries found (can be zero)
  97         :raise: DbException on error
  98         """
  99         raise DbException("Method 'count' not implemented")
 100
 101     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 102         """
 103         Obtain one entry matching q_filter
 104         :param table: collection or table
 105         :param q_filter: Filter
 106         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 107         it raises a DbException
 108         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 109         that it raises a DbException
 110         :return: The requested element, or None
 111         """
 112         raise DbException("Method 'get_one' not implemented")
 113
 114     def del_list(self, table, q_filter=None):
 115         """
 116         Deletes all entries that match q_filter
 117         :param table: collection or table
 118         :param q_filter: Filter
 119         :return: Dict with the number of entries deleted
 120         """
 121         raise DbException("Method 'del_list' not implemented")
 122
 123     def del_one(self, table, q_filter=None, fail_on_empty=True):
 124         """
 125         Deletes one entry that matches q_filter
 126         :param table: collection or table
 127         :param q_filter: Filter
 128         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 129         which case it raises a DbException
 130         :return: Dict with the number of entries deleted
 131         """
 132         raise DbException("Method 'del_one' not implemented")
 133
 134     def create(self, table, indata):
 135         """
 136         Add a new entry at database
 137         :param table: collection or table
 138         :param indata: content to be added
 139         :return: database id of the inserted element. Raises a DbException on error
 140         """
 141         raise DbException("Method 'create' not implemented")
 142
 143     def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None):
 144         """
 145         Modifies an entry at database
 146         :param table: collection or table
 147         :param q_filter: Filter
 148         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 149         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 150         it raises a DbException
 151         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 152                       ignored. If not exist, it is ignored
 153         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 154                      if exist in the array is removed. If not exist, it is ignored
 155         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 156                      is appended to the end of the array
 157         :return: Dict with the number of entries modified. None if no matching is found.
 158         """
 159         raise DbException("Method 'set_one' not implemented")
 160
 161     def set_list(self, table, q_filter, update_dict):
 162         """
 163         Modifies al matching entries at database
 164         :param table: collection or table
 165         :param q_filter: Filter
 166         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 167         :return: Dict with the number of entries modified
 168         """
 169         raise DbException("Method 'set_list' not implemented")
 170
 171     def replace(self, table, _id, indata, fail_on_empty=True):
 172         """
 173         Replace the content of an entry
 174         :param table: collection or table
 175         :param _id: internal database id
 176         :param indata: content to replace
 177         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 178         it raises a DbException
 179         :return: Dict with the number of entries replaced
 180         """
 181         raise DbException("Method 'replace' not implemented")
 182
 183     def _join_secret_key(self, update_key):
 184         """
 185         Returns a xor byte combination of the internal secret_key and the provided update_key.
 186         It does not modify the internal secret_key. Used for adding salt, join keys, etc.
 187         :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
 188         :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
 189                  are None
 190         """
 191         if not update_key:
 192             return self.secret_key
 193         elif isinstance(update_key, str):
 194             update_key_bytes = update_key.encode()
 195         else:
 196             update_key_bytes = update_key
 197
 198         new_secret_key = bytearray(self.secret_key) if self.secret_key else bytearray(32)
 199         for i, b in enumerate(update_key_bytes):
 200             new_secret_key[i % 32] ^= b
 201         return bytes(new_secret_key)
 202
 203     def set_secret_key(self, new_secret_key, replace=False):
 204         """
 205         Updates internal secret_key used for encryption, with a byte xor
 206         :param new_secret_key: string or byte array. It is recommended a 32 byte length
 207         :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
 208         :return: None
 209         """
 210         if replace:
 211             self.secret_key = None
 212         self.secret_key = self._join_secret_key(new_secret_key)
 213
 214     def encrypt(self, value, schema_version=None, salt=None):
 215         """
 216         Encrypt a value
 217         :param value: value to be encrypted. It is string/unicode
 218         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 219                If '1.1' symmetric AES encryption is done
 220         :param salt: optional salt to be used. Must be str
 221         :return: Encrypted content of value
 222         """
 223         if not self.secret_key or not schema_version or schema_version == '1.0':
 224             return value
 225         else:
 226             secret_key = self._join_secret_key(salt)
 227             cipher = AES.new(secret_key)
 228             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 229             encrypted_msg = cipher.encrypt(padded_private_msg)
 230             encoded_encrypted_msg = b64encode(encrypted_msg)
 231             return encoded_encrypted_msg.decode("ascii")
 232
 233     def decrypt(self, value, schema_version=None, salt=None):
 234         """
 235         Decrypt an encrypted value
 236         :param value: value to be decrypted. It is a base64 string
 237         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 238                If '1.1' symmetric AES encryption has been done
 239         :param salt: optional salt to be used
 240         :return: Plain content of value
 241         """
 242         if not self.secret_key or not schema_version or schema_version == '1.0':
 243             return value
 244         else:
 245             secret_key = self._join_secret_key(salt)
 246             encrypted_msg = b64decode(value)
 247             cipher = AES.new(secret_key)
 248             decrypted_msg = cipher.decrypt(encrypted_msg)
 249             try:
 250                 unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 251             except UnicodeDecodeError:
 252                 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
 253                                   http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 254             return unpadded_private_msg
 255
 256
 257 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 258     """
 259     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 260     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 261     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 262     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 263         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 264                     the element of the array is deleted, otherwise it is edited.
 265         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 266         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 267         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 268                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 269                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 270         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 271                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 272                     format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
 273                     found. If value is None the matched items are deleted, otherwise they are edited.
 274         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 275                     is changed. A value of None has not sense.
 276         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 277                     nothing is changed. A value of None has not sense.
 278     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 279         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 280         insertions must take into account the deleted items.
 281     :param dict_to_change:  Target dictionary to be changed.
 282     :param dict_reference: Dictionary that contains changes to be applied.
 283     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 284     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 285     """
 286     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 287         to_append = {}
 288         to_insert_at_index = {}
 289         values_to_edit_delete = {}
 290         indexes_to_edit_delete = []
 291         array_edition = None
 292         _key_list.append("")
 293         for k in _dict_reference:
 294             _key_list[-1] = str(k)
 295             if not isinstance(k, str) or not k.startswith("$"):
 296                 if array_edition is True:
 297                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 298                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 299                 array_edition = False
 300                 continue
 301             else:
 302                 if array_edition is False:
 303                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 304                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 305                 array_edition = True
 306             insert = False
 307             indexes = []  # indexes to edit or insert
 308             kitem = k[1:]
 309             if kitem.startswith('+'):
 310                 insert = True
 311                 kitem = kitem[1:]
 312                 if _dict_reference[k] is None:
 313                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 314                         ":".join(_key_list)))
 315
 316             if kitem.startswith('[') and kitem.endswith(']'):
 317                 try:
 318                     index = int(kitem[1:-1])
 319                     if index < 0:
 320                         index += len(array_to_change)
 321                     if index < 0:
 322                         index = 0  # skip outside index edition
 323                     indexes.append(index)
 324                 except Exception:
 325                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 326                         ":".join(_key_list)))
 327             elif kitem:
 328                 # match_found_skip = False
 329                 try:
 330                     filter_in = yaml.safe_load(kitem)
 331                 except Exception:
 332                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 333                 if isinstance(filter_in, dict):
 334                     for index, item in enumerate(array_to_change):
 335                         for filter_k, filter_v in filter_in.items():
 336                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 337                                 break
 338                         else:  # match found
 339                             if insert:
 340                                 # match_found_skip = True
 341                                 insert = False
 342                                 break
 343                             else:
 344                                 indexes.append(index)
 345                 else:
 346                     index = 0
 347                     try:
 348                         while True:  # if not match a ValueError exception will be raise
 349                             index = array_to_change.index(filter_in, index)
 350                             if insert:
 351                                 # match_found_skip = True
 352                                 insert = False
 353                                 break
 354                             indexes.append(index)
 355                             index += 1
 356                     except ValueError:
 357                         pass
 358
 359                 # if match_found_skip:
 360                 #     continue
 361             elif not insert:
 362                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 363                     ":".join(_key_list)))
 364             for index in indexes:
 365                 if insert:
 366                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 367                         # Several different insertions on the same item of the array
 368                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 369                             ":".join(_key_list), index))
 370                     to_insert_at_index[index] = _dict_reference[k]
 371                 else:
 372                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 373                         # Several different editions on the same item of the array
 374                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 375                             ":".join(_key_list), index))
 376                     indexes_to_edit_delete.append(index)
 377                     values_to_edit_delete[index] = _dict_reference[k]
 378             if not indexes:
 379                 if insert:
 380                     to_append[k] = _dict_reference[k]
 381                 # elif _dict_reference[k] is not None:
 382                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 383                 #         ":".join(_key_list)))
 384
 385         # edition/deletion is done before insertion
 386         indexes_to_edit_delete.sort(reverse=True)
 387         for index in indexes_to_edit_delete:
 388             _key_list[-1] = str(index)
 389             try:
 390                 if values_to_edit_delete[index] is None:  # None->Anything
 391                     try:
 392                         del (array_to_change[index])
 393                     except IndexError:
 394                         pass  # it is not consider an error if this index does not exist
 395                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 396                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 397                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 398                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 399                 else:  # Dict->NotDict
 400                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 401                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 402                             continue
 403                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 404                     # calling deep_update_rfc7396 to delete the None values
 405                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 406             except IndexError:
 407                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 408
 409         # insertion with indexes
 410         to_insert_indexes = list(to_insert_at_index.keys())
 411         to_insert_indexes.sort(reverse=True)
 412         for index in to_insert_indexes:
 413             array_to_change.insert(index, to_insert_at_index[index])
 414
 415         # append
 416         for k, insert_value in to_append.items():
 417             _key_list[-1] = str(k)
 418             insert_value_copy = deepcopy(insert_value)
 419             if isinstance(insert_value_copy, dict):
 420                 # calling deep_update_rfc7396 to delete the None values
 421                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 422             array_to_change.append(insert_value_copy)
 423
 424         _key_list.pop()
 425         if array_edition:
 426             return True
 427         return False
 428
 429     if key_list is None:
 430         key_list = []
 431     key_list.append("")
 432     for k in dict_reference:
 433         key_list[-1] = str(k)
 434         if dict_reference[k] is None:   # None->Anything
 435             if k in dict_to_change:
 436                 del dict_to_change[k]
 437         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 438             dict_to_change[k] = deepcopy(dict_reference[k])
 439         elif k not in dict_to_change:  # Dict->Empty
 440             dict_to_change[k] = deepcopy(dict_reference[k])
 441             # calling deep_update_rfc7396 to delete the None values
 442             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 443         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 444             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 445         else:       # Dict->NotDict
 446             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 447                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 448                     continue
 449             dict_to_change[k] = deepcopy(dict_reference[k])
 450             # calling deep_update_rfc7396 to delete the None values
 451             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 452     key_list.pop()
 453
 454
 455 def deep_update(dict_to_change, dict_reference):
 456     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 457     return deep_update_rfc7396(dict_to_change, dict_reference)