osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 import re
  21 from http import HTTPStatus
  22 from copy import deepcopy
  23 from Crypto.Cipher import AES
  24 from base64 import b64decode, b64encode
  25 from osm_common.common_utils import FakeLock
  26 from threading import Lock
  27
  28 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  29
  30
  31 class DbException(Exception):
  32
  33     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  34         self.http_code = http_code
  35         Exception.__init__(self, "database exception " + str(message))
  36
  37
  38 class DbBase(object):
  39
  40     def __init__(self, logger_name='db', lock=False):
  41         """
  42         Constructor of dbBase
  43         :param logger_name: logging name
  44         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  45             False, None: Do not protect, this object will only be accessed by one thread
  46             True: This object needs to be protected by several threads accessing.
  47             Lock object. Use thi Lock for the threads access protection
  48         """
  49         self.logger = logging.getLogger(logger_name)
  50         self.secret_key = None  # 32 bytes length array used for encrypt/decrypt
  51         if not lock:
  52             self.lock = FakeLock()
  53         elif lock is True:
  54             self.lock = Lock()
  55         elif isinstance(lock, Lock):
  56             self.lock = lock
  57         else:
  58             raise ValueError("lock parameter must be a Lock classclass or boolean")
  59
  60     def db_connect(self, config, target_version=None):
  61         """
  62         Connect to database
  63         :param config: Configuration of database. Contains among others:
  64             host:   database host (mandatory)
  65             port:   database port (mandatory)
  66             name:   database name (mandatory)
  67             user:   database username
  68             password:   database password
  69             commonkey: common OSM key used for sensible information encryption
  70             materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
  71         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  72         :return: None or raises DbException on error
  73         """
  74         raise DbException("Method 'db_connect' not implemented")
  75
  76     def db_disconnect(self):
  77         """
  78         Disconnect from database
  79         :return: None
  80         """
  81         pass
  82
  83     def get_list(self, table, q_filter=None):
  84         """
  85         Obtain a list of entries matching q_filter
  86         :param table: collection or table
  87         :param q_filter: Filter
  88         :return: a list (can be empty) with the found entries. Raises DbException on error
  89         """
  90         raise DbException("Method 'get_list' not implemented")
  91
  92     def count(self, table, q_filter=None):
  93         """
  94         Count the number of entries matching q_filter
  95         :param table: collection or table
  96         :param q_filter: Filter
  97         :return: number of entries found (can be zero)
  98         :raise: DbException on error
  99         """
 100         raise DbException("Method 'count' not implemented")
 101
 102     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 103         """
 104         Obtain one entry matching q_filter
 105         :param table: collection or table
 106         :param q_filter: Filter
 107         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 108         it raises a DbException
 109         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 110         that it raises a DbException
 111         :return: The requested element, or None
 112         """
 113         raise DbException("Method 'get_one' not implemented")
 114
 115     def del_list(self, table, q_filter=None):
 116         """
 117         Deletes all entries that match q_filter
 118         :param table: collection or table
 119         :param q_filter: Filter
 120         :return: Dict with the number of entries deleted
 121         """
 122         raise DbException("Method 'del_list' not implemented")
 123
 124     def del_one(self, table, q_filter=None, fail_on_empty=True):
 125         """
 126         Deletes one entry that matches q_filter
 127         :param table: collection or table
 128         :param q_filter: Filter
 129         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 130         which case it raises a DbException
 131         :return: Dict with the number of entries deleted
 132         """
 133         raise DbException("Method 'del_one' not implemented")
 134
 135     def create(self, table, indata):
 136         """
 137         Add a new entry at database
 138         :param table: collection or table
 139         :param indata: content to be added
 140         :return: database '_id' of the inserted element. Raises a DbException on error
 141         """
 142         raise DbException("Method 'create' not implemented")
 143
 144     def create_list(self, table, indata_list):
 145         """
 146         Add several entries at once
 147         :param table: collection or table
 148         :param indata_list: list of elements to insert. Each element must be a dictionary.
 149             An '_id' key based on random uuid is added at each element if missing
 150         :return: list of inserted '_id's. Exception on error
 151         """
 152         raise DbException("Method 'create_list' not implemented")
 153
 154     def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None,
 155                 push_list=None, pull_list=None):
 156         """
 157         Modifies an entry at database
 158         :param table: collection or table
 159         :param q_filter: Filter
 160         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 161         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 162         it raises a DbException
 163         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 164                       ignored. If not exist, it is ignored
 165         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 166                      if exist in the array is removed. If not exist, it is ignored
 167         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 168                      is appended to the end of the array
 169         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 170         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 171                           whole array
 172         :return: Dict with the number of entries modified. None if no matching is found.
 173         """
 174         raise DbException("Method 'set_one' not implemented")
 175
 176     def set_list(self, table, q_filter, update_dict, unset=None, pull=None, push=None, push_list=None, pull_list=None):
 177         """
 178         Modifies al matching entries at database
 179         :param table: collection or table
 180         :param q_filter: Filter
 181         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 182         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 183                       ignored. If not exist, it is ignored
 184         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 185                      if exist in the array is removed. If not exist, it is ignored
 186         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 187                      is appended to the end of the array
 188         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 189         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 190                           whole array
 191         :return: Dict with the number of entries modified
 192         """
 193         raise DbException("Method 'set_list' not implemented")
 194
 195     def replace(self, table, _id, indata, fail_on_empty=True):
 196         """
 197         Replace the content of an entry
 198         :param table: collection or table
 199         :param _id: internal database id
 200         :param indata: content to replace
 201         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 202         it raises a DbException
 203         :return: Dict with the number of entries replaced
 204         """
 205         raise DbException("Method 'replace' not implemented")
 206
 207     def _join_secret_key(self, update_key):
 208         """
 209         Returns a xor byte combination of the internal secret_key and the provided update_key.
 210         It does not modify the internal secret_key. Used for adding salt, join keys, etc.
 211         :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
 212         :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
 213                  are None
 214         """
 215         if not update_key:
 216             return self.secret_key
 217         elif isinstance(update_key, str):
 218             update_key_bytes = update_key.encode()
 219         else:
 220             update_key_bytes = update_key
 221
 222         new_secret_key = bytearray(self.secret_key) if self.secret_key else bytearray(32)
 223         for i, b in enumerate(update_key_bytes):
 224             new_secret_key[i % 32] ^= b
 225         return bytes(new_secret_key)
 226
 227     def set_secret_key(self, new_secret_key, replace=False):
 228         """
 229         Updates internal secret_key used for encryption, with a byte xor
 230         :param new_secret_key: string or byte array. It is recommended a 32 byte length
 231         :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
 232         :return: None
 233         """
 234         if replace:
 235             self.secret_key = None
 236         self.secret_key = self._join_secret_key(new_secret_key)
 237
 238     def get_secret_key(self):
 239         """
 240         Get the database secret key in case it is not done when "connect" is called. It can happens when database is
 241         empty after an initial install. It should skip if secret is already obtained.
 242         """
 243         pass
 244
 245     def encrypt(self, value, schema_version=None, salt=None):
 246         """
 247         Encrypt a value
 248         :param value: value to be encrypted. It is string/unicode
 249         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 250                If '1.1' symmetric AES encryption is done
 251         :param salt: optional salt to be used. Must be str
 252         :return: Encrypted content of value
 253         """
 254         self.get_secret_key()
 255         if not self.secret_key or not schema_version or schema_version == '1.0':
 256             return value
 257         else:
 258             secret_key = self._join_secret_key(salt)
 259             cipher = AES.new(secret_key)
 260             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 261             encrypted_msg = cipher.encrypt(padded_private_msg)
 262             encoded_encrypted_msg = b64encode(encrypted_msg)
 263             return encoded_encrypted_msg.decode("ascii")
 264
 265     def decrypt(self, value, schema_version=None, salt=None):
 266         """
 267         Decrypt an encrypted value
 268         :param value: value to be decrypted. It is a base64 string
 269         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 270                If '1.1' symmetric AES encryption has been done
 271         :param salt: optional salt to be used
 272         :return: Plain content of value
 273         """
 274         self.get_secret_key()
 275         if not self.secret_key or not schema_version or schema_version == '1.0':
 276             return value
 277         else:
 278             secret_key = self._join_secret_key(salt)
 279             encrypted_msg = b64decode(value)
 280             cipher = AES.new(secret_key)
 281             decrypted_msg = cipher.decrypt(encrypted_msg)
 282             try:
 283                 unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 284             except UnicodeDecodeError:
 285                 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
 286                                   http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 287             return unpadded_private_msg
 288
 289     def encrypt_decrypt_fields(self, item, action, fields=None, flags=None, schema_version=None, salt=None):
 290         if not fields:
 291             return
 292         self.get_secret_key()
 293         actions = ['encrypt', 'decrypt']
 294         if action.lower() not in actions:
 295             raise DbException("Unknown action ({}): Must be one of {}".format(action, actions),
 296                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 297         method = self.encrypt if action.lower() == 'encrypt' else self.decrypt
 298         if flags is None:
 299             flags = re.I
 300
 301         def process(_item):
 302             if isinstance(_item, list):
 303                 for elem in _item:
 304                     process(elem)
 305             elif isinstance(_item, dict):
 306                 for key, val in _item.items():
 307                     if isinstance(val, str):
 308                         if any(re.search(f, key, flags) for f in fields):
 309                             _item[key] = method(val, schema_version, salt)
 310                     else:
 311                         process(val)
 312         process(item)
 313
 314
 315 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 316     """
 317     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 318     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 319     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 320     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 321         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 322                     the element of the array is deleted, otherwise it is edited.
 323         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 324         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 325         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 326                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 327                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 328         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 329                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 330                     format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
 331                     found. If value is None the matched items are deleted, otherwise they are edited.
 332         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 333                     is changed. A value of None has not sense.
 334         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 335                     nothing is changed. A value of None has not sense.
 336     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 337         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 338         insertions must take into account the deleted items.
 339     :param dict_to_change:  Target dictionary to be changed.
 340     :param dict_reference: Dictionary that contains changes to be applied.
 341     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 342     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 343     """
 344     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 345         to_append = {}
 346         to_insert_at_index = {}
 347         values_to_edit_delete = {}
 348         indexes_to_edit_delete = []
 349         array_edition = None
 350         _key_list.append("")
 351         for k in _dict_reference:
 352             _key_list[-1] = str(k)
 353             if not isinstance(k, str) or not k.startswith("$"):
 354                 if array_edition is True:
 355                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 356                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 357                 array_edition = False
 358                 continue
 359             else:
 360                 if array_edition is False:
 361                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 362                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 363                 array_edition = True
 364             insert = False
 365             indexes = []  # indexes to edit or insert
 366             kitem = k[1:]
 367             if kitem.startswith('+'):
 368                 insert = True
 369                 kitem = kitem[1:]
 370                 if _dict_reference[k] is None:
 371                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 372                         ":".join(_key_list)))
 373
 374             if kitem.startswith('[') and kitem.endswith(']'):
 375                 try:
 376                     index = int(kitem[1:-1])
 377                     if index < 0:
 378                         index += len(array_to_change)
 379                     if index < 0:
 380                         index = 0  # skip outside index edition
 381                     indexes.append(index)
 382                 except Exception:
 383                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 384                         ":".join(_key_list)))
 385             elif kitem:
 386                 # match_found_skip = False
 387                 try:
 388                     filter_in = yaml.safe_load(kitem)
 389                 except Exception:
 390                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 391                 if isinstance(filter_in, dict):
 392                     for index, item in enumerate(array_to_change):
 393                         for filter_k, filter_v in filter_in.items():
 394                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 395                                 break
 396                         else:  # match found
 397                             if insert:
 398                                 # match_found_skip = True
 399                                 insert = False
 400                                 break
 401                             else:
 402                                 indexes.append(index)
 403                 else:
 404                     index = 0
 405                     try:
 406                         while True:  # if not match a ValueError exception will be raise
 407                             index = array_to_change.index(filter_in, index)
 408                             if insert:
 409                                 # match_found_skip = True
 410                                 insert = False
 411                                 break
 412                             indexes.append(index)
 413                             index += 1
 414                     except ValueError:
 415                         pass
 416
 417                 # if match_found_skip:
 418                 #     continue
 419             elif not insert:
 420                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 421                     ":".join(_key_list)))
 422             for index in indexes:
 423                 if insert:
 424                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 425                         # Several different insertions on the same item of the array
 426                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 427                             ":".join(_key_list), index))
 428                     to_insert_at_index[index] = _dict_reference[k]
 429                 else:
 430                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 431                         # Several different editions on the same item of the array
 432                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 433                             ":".join(_key_list), index))
 434                     indexes_to_edit_delete.append(index)
 435                     values_to_edit_delete[index] = _dict_reference[k]
 436             if not indexes:
 437                 if insert:
 438                     to_append[k] = _dict_reference[k]
 439                 # elif _dict_reference[k] is not None:
 440                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 441                 #         ":".join(_key_list)))
 442
 443         # edition/deletion is done before insertion
 444         indexes_to_edit_delete.sort(reverse=True)
 445         for index in indexes_to_edit_delete:
 446             _key_list[-1] = str(index)
 447             try:
 448                 if values_to_edit_delete[index] is None:  # None->Anything
 449                     try:
 450                         del (array_to_change[index])
 451                     except IndexError:
 452                         pass  # it is not consider an error if this index does not exist
 453                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 454                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 455                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 456                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 457                 else:  # Dict->NotDict
 458                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 459                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 460                             continue
 461                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 462                     # calling deep_update_rfc7396 to delete the None values
 463                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 464             except IndexError:
 465                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 466
 467         # insertion with indexes
 468         to_insert_indexes = list(to_insert_at_index.keys())
 469         to_insert_indexes.sort(reverse=True)
 470         for index in to_insert_indexes:
 471             array_to_change.insert(index, to_insert_at_index[index])
 472
 473         # append
 474         for k, insert_value in to_append.items():
 475             _key_list[-1] = str(k)
 476             insert_value_copy = deepcopy(insert_value)
 477             if isinstance(insert_value_copy, dict):
 478                 # calling deep_update_rfc7396 to delete the None values
 479                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 480             array_to_change.append(insert_value_copy)
 481
 482         _key_list.pop()
 483         if array_edition:
 484             return True
 485         return False
 486
 487     if key_list is None:
 488         key_list = []
 489     key_list.append("")
 490     for k in dict_reference:
 491         key_list[-1] = str(k)
 492         if dict_reference[k] is None:   # None->Anything
 493             if k in dict_to_change:
 494                 del dict_to_change[k]
 495         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 496             dict_to_change[k] = deepcopy(dict_reference[k])
 497         elif k not in dict_to_change:  # Dict->Empty
 498             dict_to_change[k] = deepcopy(dict_reference[k])
 499             # calling deep_update_rfc7396 to delete the None values
 500             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 501         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 502             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 503         else:       # Dict->NotDict
 504             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 505                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 506                     continue
 507             dict_to_change[k] = deepcopy(dict_reference[k])
 508             # calling deep_update_rfc7396 to delete the None values
 509             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 510     key_list.pop()
 511
 512
 513 def deep_update(dict_to_change, dict_reference):
 514     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 515     return deep_update_rfc7396(dict_to_change, dict_reference)