osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 import re
  21 from http import HTTPStatus
  22 from copy import deepcopy
  23 from Crypto.Cipher import AES
  24 from base64 import b64decode, b64encode
  25 from osm_common.common_utils import FakeLock
  26 from threading import Lock
  27
  28 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  29
  30
  31 class DbException(Exception):
  32
  33     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  34         self.http_code = http_code
  35         Exception.__init__(self, "database exception " + str(message))
  36
  37
  38 class DbBase(object):
  39
  40     def __init__(self, logger_name='db', lock=False):
  41         """
  42         Constructor of dbBase
  43         :param logger_name: logging name
  44         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  45             False, None: Do not protect, this object will only be accessed by one thread
  46             True: This object needs to be protected by several threads accessing.
  47             Lock object. Use thi Lock for the threads access protection
  48         """
  49         self.logger = logging.getLogger(logger_name)
  50         self.secret_key = None  # 32 bytes length array used for encrypt/decrypt
  51         if not lock:
  52             self.lock = FakeLock()
  53         elif lock is True:
  54             self.lock = Lock()
  55         elif isinstance(lock, Lock):
  56             self.lock = lock
  57         else:
  58             raise ValueError("lock parameter must be a Lock classclass or boolean")
  59
  60     def db_connect(self, config, target_version=None):
  61         """
  62         Connect to database
  63         :param config: Configuration of database. Contains among others:
  64             host:   database host (mandatory)
  65             port:   database port (mandatory)
  66             name:   database name (mandatory)
  67             user:   database username
  68             password:   database password
  69             commonkey: common OSM key used for sensible information encryption
  70             materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
  71         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  72         :return: None or raises DbException on error
  73         """
  74         raise DbException("Method 'db_connect' not implemented")
  75
  76     def db_disconnect(self):
  77         """
  78         Disconnect from database
  79         :return: None
  80         """
  81         pass
  82
  83     def get_list(self, table, q_filter=None):
  84         """
  85         Obtain a list of entries matching q_filter
  86         :param table: collection or table
  87         :param q_filter: Filter
  88         :return: a list (can be empty) with the found entries. Raises DbException on error
  89         """
  90         raise DbException("Method 'get_list' not implemented")
  91
  92     def count(self, table, q_filter=None):
  93         """
  94         Count the number of entries matching q_filter
  95         :param table: collection or table
  96         :param q_filter: Filter
  97         :return: number of entries found (can be zero)
  98         :raise: DbException on error
  99         """
 100         raise DbException("Method 'count' not implemented")
 101
 102     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 103         """
 104         Obtain one entry matching q_filter
 105         :param table: collection or table
 106         :param q_filter: Filter
 107         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 108         it raises a DbException
 109         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 110         that it raises a DbException
 111         :return: The requested element, or None
 112         """
 113         raise DbException("Method 'get_one' not implemented")
 114
 115     def del_list(self, table, q_filter=None):
 116         """
 117         Deletes all entries that match q_filter
 118         :param table: collection or table
 119         :param q_filter: Filter
 120         :return: Dict with the number of entries deleted
 121         """
 122         raise DbException("Method 'del_list' not implemented")
 123
 124     def del_one(self, table, q_filter=None, fail_on_empty=True):
 125         """
 126         Deletes one entry that matches q_filter
 127         :param table: collection or table
 128         :param q_filter: Filter
 129         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 130         which case it raises a DbException
 131         :return: Dict with the number of entries deleted
 132         """
 133         raise DbException("Method 'del_one' not implemented")
 134
 135     def create(self, table, indata):
 136         """
 137         Add a new entry at database
 138         :param table: collection or table
 139         :param indata: content to be added
 140         :return: database id of the inserted element. Raises a DbException on error
 141         """
 142         raise DbException("Method 'create' not implemented")
 143
 144     def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None):
 145         """
 146         Modifies an entry at database
 147         :param table: collection or table
 148         :param q_filter: Filter
 149         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 150         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 151         it raises a DbException
 152         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 153                       ignored. If not exist, it is ignored
 154         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 155                      if exist in the array is removed. If not exist, it is ignored
 156         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 157                      is appended to the end of the array
 158         :return: Dict with the number of entries modified. None if no matching is found.
 159         """
 160         raise DbException("Method 'set_one' not implemented")
 161
 162     def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None):
 163         """
 164         Modifies al matching entries at database
 165         :param table: collection or table
 166         :param q_filter: Filter
 167         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 168         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 169                       ignored. If not exist, it is ignored
 170         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 171                      if exist in the array is removed. If not exist, it is ignored
 172         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 173                      is appended to the end of the array
 174         :return: Dict with the number of entries modified
 175         """
 176         raise DbException("Method 'set_list' not implemented")
 177
 178     def replace(self, table, _id, indata, fail_on_empty=True):
 179         """
 180         Replace the content of an entry
 181         :param table: collection or table
 182         :param _id: internal database id
 183         :param indata: content to replace
 184         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 185         it raises a DbException
 186         :return: Dict with the number of entries replaced
 187         """
 188         raise DbException("Method 'replace' not implemented")
 189
 190     def _join_secret_key(self, update_key):
 191         """
 192         Returns a xor byte combination of the internal secret_key and the provided update_key.
 193         It does not modify the internal secret_key. Used for adding salt, join keys, etc.
 194         :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
 195         :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
 196                  are None
 197         """
 198         if not update_key:
 199             return self.secret_key
 200         elif isinstance(update_key, str):
 201             update_key_bytes = update_key.encode()
 202         else:
 203             update_key_bytes = update_key
 204
 205         new_secret_key = bytearray(self.secret_key) if self.secret_key else bytearray(32)
 206         for i, b in enumerate(update_key_bytes):
 207             new_secret_key[i % 32] ^= b
 208         return bytes(new_secret_key)
 209
 210     def set_secret_key(self, new_secret_key, replace=False):
 211         """
 212         Updates internal secret_key used for encryption, with a byte xor
 213         :param new_secret_key: string or byte array. It is recommended a 32 byte length
 214         :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
 215         :return: None
 216         """
 217         if replace:
 218             self.secret_key = None
 219         self.secret_key = self._join_secret_key(new_secret_key)
 220
 221     def get_secret_key(self):
 222         """
 223         Get the database secret key in case it is not done when "connect" is called. It can happens when database is
 224         empty after an initial install. It should skip if secret is already obtained.
 225         """
 226         pass
 227
 228     def encrypt(self, value, schema_version=None, salt=None):
 229         """
 230         Encrypt a value
 231         :param value: value to be encrypted. It is string/unicode
 232         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 233                If '1.1' symmetric AES encryption is done
 234         :param salt: optional salt to be used. Must be str
 235         :return: Encrypted content of value
 236         """
 237         self.get_secret_key()
 238         if not self.secret_key or not schema_version or schema_version == '1.0':
 239             return value
 240         else:
 241             secret_key = self._join_secret_key(salt)
 242             cipher = AES.new(secret_key)
 243             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 244             encrypted_msg = cipher.encrypt(padded_private_msg)
 245             encoded_encrypted_msg = b64encode(encrypted_msg)
 246             return encoded_encrypted_msg.decode("ascii")
 247
 248     def decrypt(self, value, schema_version=None, salt=None):
 249         """
 250         Decrypt an encrypted value
 251         :param value: value to be decrypted. It is a base64 string
 252         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 253                If '1.1' symmetric AES encryption has been done
 254         :param salt: optional salt to be used
 255         :return: Plain content of value
 256         """
 257         self.get_secret_key()
 258         if not self.secret_key or not schema_version or schema_version == '1.0':
 259             return value
 260         else:
 261             secret_key = self._join_secret_key(salt)
 262             encrypted_msg = b64decode(value)
 263             cipher = AES.new(secret_key)
 264             decrypted_msg = cipher.decrypt(encrypted_msg)
 265             try:
 266                 unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 267             except UnicodeDecodeError:
 268                 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
 269                                   http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 270             return unpadded_private_msg
 271
 272     def encrypt_decrypt_fields(self, item, action, fields=None, flags=re.I, schema_version=None, salt=None):
 273         if not fields:
 274             return
 275         self.get_secret_key()
 276         actions = ['encrypt', 'decrypt']
 277         if action.lower() not in actions:
 278             raise DbException("Unknown action ({}): Must be one of {}".format(action, actions),
 279                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 280         method = self.encrypt if action.lower() == 'encrypt' else self.decrypt
 281
 282         def process(item):
 283             if isinstance(item, list):
 284                 for elem in item:
 285                     process(elem)
 286             elif isinstance(item, dict):
 287                 for key, val in item.items():
 288                     if any(re.search(f, key, flags) for f in fields) and isinstance(val, str):
 289                         item[key] = method(val, schema_version, salt)
 290                     else:
 291                         process(val)
 292
 293         process(item)
 294
 295
 296 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 297     """
 298     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 299     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 300     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 301     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 302         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 303                     the element of the array is deleted, otherwise it is edited.
 304         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 305         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 306         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 307                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 308                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 309         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 310                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 311                     format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
 312                     found. If value is None the matched items are deleted, otherwise they are edited.
 313         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 314                     is changed. A value of None has not sense.
 315         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 316                     nothing is changed. A value of None has not sense.
 317     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 318         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 319         insertions must take into account the deleted items.
 320     :param dict_to_change:  Target dictionary to be changed.
 321     :param dict_reference: Dictionary that contains changes to be applied.
 322     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 323     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 324     """
 325     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 326         to_append = {}
 327         to_insert_at_index = {}
 328         values_to_edit_delete = {}
 329         indexes_to_edit_delete = []
 330         array_edition = None
 331         _key_list.append("")
 332         for k in _dict_reference:
 333             _key_list[-1] = str(k)
 334             if not isinstance(k, str) or not k.startswith("$"):
 335                 if array_edition is True:
 336                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 337                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 338                 array_edition = False
 339                 continue
 340             else:
 341                 if array_edition is False:
 342                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 343                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 344                 array_edition = True
 345             insert = False
 346             indexes = []  # indexes to edit or insert
 347             kitem = k[1:]
 348             if kitem.startswith('+'):
 349                 insert = True
 350                 kitem = kitem[1:]
 351                 if _dict_reference[k] is None:
 352                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 353                         ":".join(_key_list)))
 354
 355             if kitem.startswith('[') and kitem.endswith(']'):
 356                 try:
 357                     index = int(kitem[1:-1])
 358                     if index < 0:
 359                         index += len(array_to_change)
 360                     if index < 0:
 361                         index = 0  # skip outside index edition
 362                     indexes.append(index)
 363                 except Exception:
 364                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 365                         ":".join(_key_list)))
 366             elif kitem:
 367                 # match_found_skip = False
 368                 try:
 369                     filter_in = yaml.safe_load(kitem)
 370                 except Exception:
 371                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 372                 if isinstance(filter_in, dict):
 373                     for index, item in enumerate(array_to_change):
 374                         for filter_k, filter_v in filter_in.items():
 375                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 376                                 break
 377                         else:  # match found
 378                             if insert:
 379                                 # match_found_skip = True
 380                                 insert = False
 381                                 break
 382                             else:
 383                                 indexes.append(index)
 384                 else:
 385                     index = 0
 386                     try:
 387                         while True:  # if not match a ValueError exception will be raise
 388                             index = array_to_change.index(filter_in, index)
 389                             if insert:
 390                                 # match_found_skip = True
 391                                 insert = False
 392                                 break
 393                             indexes.append(index)
 394                             index += 1
 395                     except ValueError:
 396                         pass
 397
 398                 # if match_found_skip:
 399                 #     continue
 400             elif not insert:
 401                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 402                     ":".join(_key_list)))
 403             for index in indexes:
 404                 if insert:
 405                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 406                         # Several different insertions on the same item of the array
 407                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 408                             ":".join(_key_list), index))
 409                     to_insert_at_index[index] = _dict_reference[k]
 410                 else:
 411                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 412                         # Several different editions on the same item of the array
 413                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 414                             ":".join(_key_list), index))
 415                     indexes_to_edit_delete.append(index)
 416                     values_to_edit_delete[index] = _dict_reference[k]
 417             if not indexes:
 418                 if insert:
 419                     to_append[k] = _dict_reference[k]
 420                 # elif _dict_reference[k] is not None:
 421                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 422                 #         ":".join(_key_list)))
 423
 424         # edition/deletion is done before insertion
 425         indexes_to_edit_delete.sort(reverse=True)
 426         for index in indexes_to_edit_delete:
 427             _key_list[-1] = str(index)
 428             try:
 429                 if values_to_edit_delete[index] is None:  # None->Anything
 430                     try:
 431                         del (array_to_change[index])
 432                     except IndexError:
 433                         pass  # it is not consider an error if this index does not exist
 434                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 435                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 436                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 437                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 438                 else:  # Dict->NotDict
 439                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 440                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 441                             continue
 442                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 443                     # calling deep_update_rfc7396 to delete the None values
 444                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 445             except IndexError:
 446                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 447
 448         # insertion with indexes
 449         to_insert_indexes = list(to_insert_at_index.keys())
 450         to_insert_indexes.sort(reverse=True)
 451         for index in to_insert_indexes:
 452             array_to_change.insert(index, to_insert_at_index[index])
 453
 454         # append
 455         for k, insert_value in to_append.items():
 456             _key_list[-1] = str(k)
 457             insert_value_copy = deepcopy(insert_value)
 458             if isinstance(insert_value_copy, dict):
 459                 # calling deep_update_rfc7396 to delete the None values
 460                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 461             array_to_change.append(insert_value_copy)
 462
 463         _key_list.pop()
 464         if array_edition:
 465             return True
 466         return False
 467
 468     if key_list is None:
 469         key_list = []
 470     key_list.append("")
 471     for k in dict_reference:
 472         key_list[-1] = str(k)
 473         if dict_reference[k] is None:   # None->Anything
 474             if k in dict_to_change:
 475                 del dict_to_change[k]
 476         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 477             dict_to_change[k] = deepcopy(dict_reference[k])
 478         elif k not in dict_to_change:  # Dict->Empty
 479             dict_to_change[k] = deepcopy(dict_reference[k])
 480             # calling deep_update_rfc7396 to delete the None values
 481             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 482         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 483             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 484         else:       # Dict->NotDict
 485             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 486                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 487                     continue
 488             dict_to_change[k] = deepcopy(dict_reference[k])
 489             # calling deep_update_rfc7396 to delete the None values
 490             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 491     key_list.pop()
 492
 493
 494 def deep_update(dict_to_change, dict_reference):
 495     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 496     return deep_update_rfc7396(dict_to_change, dict_reference)