osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 import re
  21 from http import HTTPStatus
  22 from copy import deepcopy
  23 from Crypto.Cipher import AES
  24 from base64 import b64decode, b64encode
  25 from osm_common.common_utils import FakeLock
  26 from threading import Lock
  27
  28 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  29
  30
  31 class DbException(Exception):
  32     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  33         self.http_code = http_code
  34         Exception.__init__(self, "database exception " + str(message))
  35
  36
  37 class DbBase(object):
  38     def __init__(self, logger_name="db", lock=False):
  39         """
  40         Constructor of dbBase
  41         :param logger_name: logging name
  42         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  43             False, None: Do not protect, this object will only be accessed by one thread
  44             True: This object needs to be protected by several threads accessing.
  45             Lock object. Use thi Lock for the threads access protection
  46         """
  47         self.logger = logging.getLogger(logger_name)
  48         self.secret_key = None  # 32 bytes length array used for encrypt/decrypt
  49         if not lock:
  50             self.lock = FakeLock()
  51         elif lock is True:
  52             self.lock = Lock()
  53         elif isinstance(lock, Lock):
  54             self.lock = lock
  55         else:
  56             raise ValueError("lock parameter must be a Lock classclass or boolean")
  57
  58     def db_connect(self, config, target_version=None):
  59         """
  60         Connect to database
  61         :param config: Configuration of database. Contains among others:
  62             host:   database host (mandatory)
  63             port:   database port (mandatory)
  64             name:   database name (mandatory)
  65             user:   database username
  66             password:   database password
  67             commonkey: common OSM key used for sensible information encryption
  68             materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
  69         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  70         :return: None or raises DbException on error
  71         """
  72         raise DbException("Method 'db_connect' not implemented")
  73
  74     def db_disconnect(self):
  75         """
  76         Disconnect from database
  77         :return: None
  78         """
  79         pass
  80
  81     def get_list(self, table, q_filter=None):
  82         """
  83         Obtain a list of entries matching q_filter
  84         :param table: collection or table
  85         :param q_filter: Filter
  86         :return: a list (can be empty) with the found entries. Raises DbException on error
  87         """
  88         raise DbException("Method 'get_list' not implemented")
  89
  90     def count(self, table, q_filter=None):
  91         """
  92         Count the number of entries matching q_filter
  93         :param table: collection or table
  94         :param q_filter: Filter
  95         :return: number of entries found (can be zero)
  96         :raise: DbException on error
  97         """
  98         raise DbException("Method 'count' not implemented")
  99
 100     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 101         """
 102         Obtain one entry matching q_filter
 103         :param table: collection or table
 104         :param q_filter: Filter
 105         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 106         it raises a DbException
 107         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 108         that it raises a DbException
 109         :return: The requested element, or None
 110         """
 111         raise DbException("Method 'get_one' not implemented")
 112
 113     def del_list(self, table, q_filter=None):
 114         """
 115         Deletes all entries that match q_filter
 116         :param table: collection or table
 117         :param q_filter: Filter
 118         :return: Dict with the number of entries deleted
 119         """
 120         raise DbException("Method 'del_list' not implemented")
 121
 122     def del_one(self, table, q_filter=None, fail_on_empty=True):
 123         """
 124         Deletes one entry that matches q_filter
 125         :param table: collection or table
 126         :param q_filter: Filter
 127         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 128         which case it raises a DbException
 129         :return: Dict with the number of entries deleted
 130         """
 131         raise DbException("Method 'del_one' not implemented")
 132
 133     def create(self, table, indata):
 134         """
 135         Add a new entry at database
 136         :param table: collection or table
 137         :param indata: content to be added
 138         :return: database '_id' of the inserted element. Raises a DbException on error
 139         """
 140         raise DbException("Method 'create' not implemented")
 141
 142     def create_list(self, table, indata_list):
 143         """
 144         Add several entries at once
 145         :param table: collection or table
 146         :param indata_list: list of elements to insert. Each element must be a dictionary.
 147             An '_id' key based on random uuid is added at each element if missing
 148         :return: list of inserted '_id's. Exception on error
 149         """
 150         raise DbException("Method 'create_list' not implemented")
 151
 152     def set_one(
 153         self,
 154         table,
 155         q_filter,
 156         update_dict,
 157         fail_on_empty=True,
 158         unset=None,
 159         pull=None,
 160         push=None,
 161         push_list=None,
 162         pull_list=None,
 163     ):
 164         """
 165         Modifies an entry at database
 166         :param table: collection or table
 167         :param q_filter: Filter
 168         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 169         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 170         it raises a DbException
 171         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 172                       ignored. If not exist, it is ignored
 173         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 174                      if exist in the array is removed. If not exist, it is ignored
 175         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 176                      is appended to the end of the array
 177         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 178         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 179                           whole array
 180         :return: Dict with the number of entries modified. None if no matching is found.
 181         """
 182         raise DbException("Method 'set_one' not implemented")
 183
 184     def set_list(
 185         self,
 186         table,
 187         q_filter,
 188         update_dict,
 189         unset=None,
 190         pull=None,
 191         push=None,
 192         push_list=None,
 193         pull_list=None,
 194     ):
 195         """
 196         Modifies al matching entries at database
 197         :param table: collection or table
 198         :param q_filter: Filter
 199         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 200         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 201                       ignored. If not exist, it is ignored
 202         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 203                      if exist in the array is removed. If not exist, it is ignored
 204         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 205                      is appended to the end of the array
 206         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 207         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 208                           whole array
 209         :return: Dict with the number of entries modified
 210         """
 211         raise DbException("Method 'set_list' not implemented")
 212
 213     def replace(self, table, _id, indata, fail_on_empty=True):
 214         """
 215         Replace the content of an entry
 216         :param table: collection or table
 217         :param _id: internal database id
 218         :param indata: content to replace
 219         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 220         it raises a DbException
 221         :return: Dict with the number of entries replaced
 222         """
 223         raise DbException("Method 'replace' not implemented")
 224
 225     def _join_secret_key(self, update_key):
 226         """
 227         Returns a xor byte combination of the internal secret_key and the provided update_key.
 228         It does not modify the internal secret_key. Used for adding salt, join keys, etc.
 229         :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
 230         :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
 231                  are None
 232         """
 233         if not update_key:
 234             return self.secret_key
 235         elif isinstance(update_key, str):
 236             update_key_bytes = update_key.encode()
 237         else:
 238             update_key_bytes = update_key
 239
 240         new_secret_key = (
 241             bytearray(self.secret_key) if self.secret_key else bytearray(32)
 242         )
 243         for i, b in enumerate(update_key_bytes):
 244             new_secret_key[i % 32] ^= b
 245         return bytes(new_secret_key)
 246
 247     def set_secret_key(self, new_secret_key, replace=False):
 248         """
 249         Updates internal secret_key used for encryption, with a byte xor
 250         :param new_secret_key: string or byte array. It is recommended a 32 byte length
 251         :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
 252         :return: None
 253         """
 254         if replace:
 255             self.secret_key = None
 256         self.secret_key = self._join_secret_key(new_secret_key)
 257
 258     def get_secret_key(self):
 259         """
 260         Get the database secret key in case it is not done when "connect" is called. It can happens when database is
 261         empty after an initial install. It should skip if secret is already obtained.
 262         """
 263         pass
 264
 265     def encrypt(self, value, schema_version=None, salt=None):
 266         """
 267         Encrypt a value
 268         :param value: value to be encrypted. It is string/unicode
 269         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 270                If '1.1' symmetric AES encryption is done
 271         :param salt: optional salt to be used. Must be str
 272         :return: Encrypted content of value
 273         """
 274         self.get_secret_key()
 275         if not self.secret_key or not schema_version or schema_version == "1.0":
 276             return value
 277         else:
 278             secret_key = self._join_secret_key(salt)
 279             cipher = AES.new(secret_key)
 280             padded_private_msg = value + ("\0" * ((16 - len(value)) % 16))
 281             encrypted_msg = cipher.encrypt(padded_private_msg)
 282             encoded_encrypted_msg = b64encode(encrypted_msg)
 283             return encoded_encrypted_msg.decode("ascii")
 284
 285     def decrypt(self, value, schema_version=None, salt=None):
 286         """
 287         Decrypt an encrypted value
 288         :param value: value to be decrypted. It is a base64 string
 289         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 290                If '1.1' symmetric AES encryption has been done
 291         :param salt: optional salt to be used
 292         :return: Plain content of value
 293         """
 294         self.get_secret_key()
 295         if not self.secret_key or not schema_version or schema_version == "1.0":
 296             return value
 297         else:
 298             secret_key = self._join_secret_key(salt)
 299             encrypted_msg = b64decode(value)
 300             cipher = AES.new(secret_key)
 301             decrypted_msg = cipher.decrypt(encrypted_msg)
 302             try:
 303                 unpadded_private_msg = decrypted_msg.decode().rstrip("\0")
 304             except UnicodeDecodeError:
 305                 raise DbException(
 306                     "Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
 307                     http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
 308                 )
 309             return unpadded_private_msg
 310
 311     def encrypt_decrypt_fields(
 312         self, item, action, fields=None, flags=None, schema_version=None, salt=None
 313     ):
 314         if not fields:
 315             return
 316         self.get_secret_key()
 317         actions = ["encrypt", "decrypt"]
 318         if action.lower() not in actions:
 319             raise DbException(
 320                 "Unknown action ({}): Must be one of {}".format(action, actions),
 321                 http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
 322             )
 323         method = self.encrypt if action.lower() == "encrypt" else self.decrypt
 324         if flags is None:
 325             flags = re.I
 326
 327         def process(_item):
 328             if isinstance(_item, list):
 329                 for elem in _item:
 330                     process(elem)
 331             elif isinstance(_item, dict):
 332                 for key, val in _item.items():
 333                     if isinstance(val, str):
 334                         if any(re.search(f, key, flags) for f in fields):
 335                             _item[key] = method(val, schema_version, salt)
 336                     else:
 337                         process(val)
 338
 339         process(item)
 340
 341
 342 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 343     """
 344     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 345     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 346     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 347     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 348         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 349                     the element of the array is deleted, otherwise it is edited.
 350         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 351         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 352         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 353                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 354                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 355         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 356                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 357                     format '{key: val, key: val, ...}'; and all of them must match. Nothing happens if no match is
 358                     found. If value is None the matched items are deleted, otherwise they are edited.
 359         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 360                     is changed. A value of None has not sense.
 361         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 362                     nothing is changed. A value of None has not sense.
 363     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 364         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 365         insertions must take into account the deleted items.
 366     :param dict_to_change:  Target dictionary to be changed.
 367     :param dict_reference: Dictionary that contains changes to be applied.
 368     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 369     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 370     """
 371
 372     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 373         to_append = {}
 374         to_insert_at_index = {}
 375         values_to_edit_delete = {}
 376         indexes_to_edit_delete = []
 377         array_edition = None
 378         _key_list.append("")
 379         for k in _dict_reference:
 380             _key_list[-1] = str(k)
 381             if not isinstance(k, str) or not k.startswith("$"):
 382                 if array_edition is True:
 383                     raise DbException(
 384                         "Found array edition (keys starting with '$') and pure dictionary edition in the"
 385                         " same dict at '{}'".format(":".join(_key_list[:-1]))
 386                     )
 387                 array_edition = False
 388                 continue
 389             else:
 390                 if array_edition is False:
 391                     raise DbException(
 392                         "Found array edition (keys starting with '$') and pure dictionary edition in the"
 393                         " same dict at '{}'".format(":".join(_key_list[:-1]))
 394                     )
 395                 array_edition = True
 396             insert = False
 397             indexes = []  # indexes to edit or insert
 398             kitem = k[1:]
 399             if kitem.startswith("+"):
 400                 insert = True
 401                 kitem = kitem[1:]
 402                 if _dict_reference[k] is None:
 403                     raise DbException(
 404                         "A value of None has not sense for insertions at '{}'".format(
 405                             ":".join(_key_list)
 406                         )
 407                     )
 408
 409             if kitem.startswith("[") and kitem.endswith("]"):
 410                 try:
 411                     index = int(kitem[1:-1])
 412                     if index < 0:
 413                         index += len(array_to_change)
 414                     if index < 0:
 415                         index = 0  # skip outside index edition
 416                     indexes.append(index)
 417                 except Exception:
 418                     raise DbException(
 419                         "Wrong format at '{}'. Expecting integer index inside quotes".format(
 420                             ":".join(_key_list)
 421                         )
 422                     )
 423             elif kitem:
 424                 # match_found_skip = False
 425                 try:
 426                     filter_in = yaml.safe_load(kitem)
 427                 except Exception:
 428                     raise DbException(
 429                         "Wrong format at '{}'. Expecting '$<yaml-format>'".format(
 430                             ":".join(_key_list)
 431                         )
 432                     )
 433                 if isinstance(filter_in, dict):
 434                     for index, item in enumerate(array_to_change):
 435                         for filter_k, filter_v in filter_in.items():
 436                             if (
 437                                 not isinstance(item, dict)
 438                                 or filter_k not in item
 439                                 or item[filter_k] != filter_v
 440                             ):
 441                                 break
 442                         else:  # match found
 443                             if insert:
 444                                 # match_found_skip = True
 445                                 insert = False
 446                                 break
 447                             else:
 448                                 indexes.append(index)
 449                 else:
 450                     index = 0
 451                     try:
 452                         while True:  # if not match a ValueError exception will be raise
 453                             index = array_to_change.index(filter_in, index)
 454                             if insert:
 455                                 # match_found_skip = True
 456                                 insert = False
 457                                 break
 458                             indexes.append(index)
 459                             index += 1
 460                     except ValueError:
 461                         pass
 462
 463                 # if match_found_skip:
 464                 #     continue
 465             elif not insert:
 466                 raise DbException(
 467                     "Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 468                         ":".join(_key_list)
 469                     )
 470                 )
 471             for index in indexes:
 472                 if insert:
 473                     if (
 474                         index in to_insert_at_index
 475                         and to_insert_at_index[index] != _dict_reference[k]
 476                     ):
 477                         # Several different insertions on the same item of the array
 478                         raise DbException(
 479                             "Conflict at '{}'. Several insertions on same array index {}".format(
 480                                 ":".join(_key_list), index
 481                             )
 482                         )
 483                     to_insert_at_index[index] = _dict_reference[k]
 484                 else:
 485                     if (
 486                         index in indexes_to_edit_delete
 487                         and values_to_edit_delete[index] != _dict_reference[k]
 488                     ):
 489                         # Several different editions on the same item of the array
 490                         raise DbException(
 491                             "Conflict at '{}'. Several editions on array index {}".format(
 492                                 ":".join(_key_list), index
 493                             )
 494                         )
 495                     indexes_to_edit_delete.append(index)
 496                     values_to_edit_delete[index] = _dict_reference[k]
 497             if not indexes:
 498                 if insert:
 499                     to_append[k] = _dict_reference[k]
 500                 # elif _dict_reference[k] is not None:
 501                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 502                 #         ":".join(_key_list)))
 503
 504         # edition/deletion is done before insertion
 505         indexes_to_edit_delete.sort(reverse=True)
 506         for index in indexes_to_edit_delete:
 507             _key_list[-1] = str(index)
 508             try:
 509                 if values_to_edit_delete[index] is None:  # None->Anything
 510                     try:
 511                         del array_to_change[index]
 512                     except IndexError:
 513                         pass  # it is not consider an error if this index does not exist
 514                 elif not isinstance(
 515                     values_to_edit_delete[index], dict
 516                 ):  # NotDict->Anything
 517                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 518                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 519                     deep_update_rfc7396(
 520                         array_to_change[index], values_to_edit_delete[index], _key_list
 521                     )
 522                 else:  # Dict->NotDict
 523                     if isinstance(
 524                         array_to_change[index], list
 525                     ):  # Dict->List. Check extra array edition
 526                         if _deep_update_array(
 527                             array_to_change[index],
 528                             values_to_edit_delete[index],
 529                             _key_list,
 530                         ):
 531                             continue
 532                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 533                     # calling deep_update_rfc7396 to delete the None values
 534                     deep_update_rfc7396(
 535                         array_to_change[index], values_to_edit_delete[index], _key_list
 536                     )
 537             except IndexError:
 538                 raise DbException(
 539                     "Array edition index out of range at '{}'".format(
 540                         ":".join(_key_list)
 541                     )
 542                 )
 543
 544         # insertion with indexes
 545         to_insert_indexes = list(to_insert_at_index.keys())
 546         to_insert_indexes.sort(reverse=True)
 547         for index in to_insert_indexes:
 548             array_to_change.insert(index, to_insert_at_index[index])
 549
 550         # append
 551         for k, insert_value in to_append.items():
 552             _key_list[-1] = str(k)
 553             insert_value_copy = deepcopy(insert_value)
 554             if isinstance(insert_value_copy, dict):
 555                 # calling deep_update_rfc7396 to delete the None values
 556                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 557             array_to_change.append(insert_value_copy)
 558
 559         _key_list.pop()
 560         if array_edition:
 561             return True
 562         return False
 563
 564     if key_list is None:
 565         key_list = []
 566     key_list.append("")
 567     for k in dict_reference:
 568         key_list[-1] = str(k)
 569         if dict_reference[k] is None:  # None->Anything
 570             if k in dict_to_change:
 571                 del dict_to_change[k]
 572         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 573             dict_to_change[k] = deepcopy(dict_reference[k])
 574         elif k not in dict_to_change:  # Dict->Empty
 575             dict_to_change[k] = deepcopy(dict_reference[k])
 576             # calling deep_update_rfc7396 to delete the None values
 577             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 578         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 579             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 580         else:  # Dict->NotDict
 581             if isinstance(
 582                 dict_to_change[k], list
 583             ):  # Dict->List. Check extra array edition
 584                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 585                     continue
 586             dict_to_change[k] = deepcopy(dict_reference[k])
 587             # calling deep_update_rfc7396 to delete the None values
 588             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 589     key_list.pop()
 590
 591
 592 def deep_update(dict_to_change, dict_reference):
 593     """Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 594     return deep_update_rfc7396(dict_to_change, dict_reference)