osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22 from Crypto.Cipher import AES
  23 from base64 import b64decode, b64encode
  24 from osm_common.common_utils import FakeLock
  25 from threading import Lock
  26
  27 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  28
  29
  30 class DbException(Exception):
  31
  32     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  33         self.http_code = http_code
  34         Exception.__init__(self, "database exception " + str(message))
  35
  36
  37 class DbBase(object):
  38
  39     def __init__(self, logger_name='db', lock=False):
  40         """
  41         Constructor of dbBase
  42         :param logger_name: logging name
  43         :param lock: Used to protect simultaneous access to the same instance class by several threads:
  44             False, None: Do not protect, this object will only be accessed by one thread
  45             True: This object needs to be protected by several threads accessing.
  46             Lock object. Use thi Lock for the threads access protection
  47         """
  48         self.logger = logging.getLogger(logger_name)
  49         self.secret_key = None  # 32 bytes length array used for encrypt/decrypt
  50         if not lock:
  51             self.lock = FakeLock()
  52         elif lock is True:
  53             self.lock = Lock()
  54         elif isinstance(lock, Lock):
  55             self.lock = lock
  56         else:
  57             raise ValueError("lock parameter must be a Lock classclass or boolean")
  58
  59     def db_connect(self, config, target_version=None):
  60         """
  61         Connect to database
  62         :param config: Configuration of database. Contains among others:
  63             host:   database hosst (mandatory)
  64             port:   database port (mandatory)
  65             name:   database name (mandatory)
  66             user:   database username
  67             password:   database password
  68             commonkey: common OSM key used for sensible information encryption
  69             materpassword: same as commonkey, for backward compatibility. Deprecated, to be removed in the future
  70         :param target_version: if provided it checks if database contains required version, raising exception otherwise.
  71         :return: None or raises DbException on error
  72         """
  73         raise DbException("Method 'db_connect' not implemented")
  74
  75     def db_disconnect(self):
  76         """
  77         Disconnect from database
  78         :return: None
  79         """
  80         pass
  81
  82     def get_list(self, table, q_filter=None):
  83         """
  84         Obtain a list of entries matching q_filter
  85         :param table: collection or table
  86         :param q_filter: Filter
  87         :return: a list (can be empty) with the found entries. Raises DbException on error
  88         """
  89         raise DbException("Method 'get_list' not implemented")
  90
  91     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
  92         """
  93         Obtain one entry matching q_filter
  94         :param table: collection or table
  95         :param q_filter: Filter
  96         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
  97         it raises a DbException
  98         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
  99         that it raises a DbException
 100         :return: The requested element, or None
 101         """
 102         raise DbException("Method 'get_one' not implemented")
 103
 104     def del_list(self, table, q_filter=None):
 105         """
 106         Deletes all entries that match q_filter
 107         :param table: collection or table
 108         :param q_filter: Filter
 109         :return: Dict with the number of entries deleted
 110         """
 111         raise DbException("Method 'del_list' not implemented")
 112
 113     def del_one(self, table, q_filter=None, fail_on_empty=True):
 114         """
 115         Deletes one entry that matches q_filter
 116         :param table: collection or table
 117         :param q_filter: Filter
 118         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 119         which case it raises a DbException
 120         :return: Dict with the number of entries deleted
 121         """
 122         raise DbException("Method 'del_one' not implemented")
 123
 124     def create(self, table, indata):
 125         """
 126         Add a new entry at database
 127         :param table: collection or table
 128         :param indata: content to be added
 129         :return: database id of the inserted element. Raises a DbException on error
 130         """
 131         raise DbException("Method 'create' not implemented")
 132
 133     def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None):
 134         """
 135         Modifies an entry at database
 136         :param table: collection or table
 137         :param q_filter: Filter
 138         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 139         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 140         it raises a DbException
 141         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 142                       ignored. If not exist, it is ignored
 143         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 144                      if exist in the array is removed. If not exist, it is ignored
 145         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 146                      is appended to the end of the array
 147         :return: Dict with the number of entries modified. None if no matching is found.
 148         """
 149         raise DbException("Method 'set_one' not implemented")
 150
 151     def set_list(self, table, q_filter, update_dict):
 152         """
 153         Modifies al matching entries at database
 154         :param table: collection or table
 155         :param q_filter: Filter
 156         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 157         :return: Dict with the number of entries modified
 158         """
 159         raise DbException("Method 'set_list' not implemented")
 160
 161     def replace(self, table, _id, indata, fail_on_empty=True):
 162         """
 163         Replace the content of an entry
 164         :param table: collection or table
 165         :param _id: internal database id
 166         :param indata: content to replace
 167         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 168         it raises a DbException
 169         :return: Dict with the number of entries replaced
 170         """
 171         raise DbException("Method 'replace' not implemented")
 172
 173     def _join_secret_key(self, update_key):
 174         """
 175         Returns a xor byte combination of the internal secret_key and the provided update_key.
 176         It does not modify the internal secret_key. Used for adding salt, join keys, etc.
 177         :param update_key: Can be a string, byte or None. Recommended a long one (e.g. 32 byte length)
 178         :return: joined key in bytes with a 32 bytes length. Can be None if both internal secret_key and update_key
 179                  are None
 180         """
 181         if not update_key:
 182             return self.secret_key
 183         elif isinstance(update_key, str):
 184             update_key_bytes = update_key.encode()
 185         else:
 186             update_key_bytes = update_key
 187
 188         new_secret_key = bytearray(self.secret_key) if self.secret_key else bytearray(32)
 189         for i, b in enumerate(update_key_bytes):
 190             new_secret_key[i % 32] ^= b
 191         return bytes(new_secret_key)
 192
 193     def set_secret_key(self, new_secret_key, replace=False):
 194         """
 195         Updates internal secret_key used for encryption, with a byte xor
 196         :param new_secret_key: string or byte array. It is recommended a 32 byte length
 197         :param replace: if True, old value of internal secret_key is ignored and replaced. If false, a byte xor is used
 198         :return: None
 199         """
 200         if replace:
 201             self.secret_key = None
 202         self.secret_key = self._join_secret_key(new_secret_key)
 203
 204     def encrypt(self, value, schema_version=None, salt=None):
 205         """
 206         Encrypt a value
 207         :param value: value to be encrypted. It is string/unicode
 208         :param schema_version: used for version control. If None or '1.0' no encryption is done.
 209                If '1.1' symmetric AES encryption is done
 210         :param salt: optional salt to be used. Must be str
 211         :return: Encrypted content of value
 212         """
 213         if not self.secret_key or not schema_version or schema_version == '1.0':
 214             return value
 215         else:
 216             secret_key = self._join_secret_key(salt)
 217             cipher = AES.new(secret_key)
 218             padded_private_msg = value + ('\0' * ((16-len(value)) % 16))
 219             encrypted_msg = cipher.encrypt(padded_private_msg)
 220             encoded_encrypted_msg = b64encode(encrypted_msg)
 221             return encoded_encrypted_msg.decode("ascii")
 222
 223     def decrypt(self, value, schema_version=None, salt=None):
 224         """
 225         Decrypt an encrypted value
 226         :param value: value to be decrypted. It is a base64 string
 227         :param schema_version: used for known encryption method used. If None or '1.0' no encryption has been done.
 228                If '1.1' symmetric AES encryption has been done
 229         :param salt: optional salt to be used
 230         :return: Plain content of value
 231         """
 232         if not self.secret_key or not schema_version or schema_version == '1.0':
 233             return value
 234         else:
 235             secret_key = self._join_secret_key(salt)
 236             encrypted_msg = b64decode(value)
 237             cipher = AES.new(secret_key)
 238             decrypted_msg = cipher.decrypt(encrypted_msg)
 239             try:
 240                 unpadded_private_msg = decrypted_msg.decode().rstrip('\0')
 241             except UnicodeDecodeError:
 242                 raise DbException("Cannot decrypt information. Are you using same COMMONKEY in all OSM components?",
 243                                   http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 244             return unpadded_private_msg
 245
 246
 247 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 248     """
 249     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 250     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 251     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 252     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 253         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 254                     the element of the array is deleted, otherwise it is edited.
 255         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 256         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 257         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 258                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 259                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 260         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 261                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 262                     format '{key: val, key: val, ...}'; and all of them mast match. Nothing happens if no match is
 263                     found. If value is None the matched items are deleted, otherwise they are edited.
 264         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 265                     is changed. A value of None has not sense.
 266         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 267                     nothing is changed. A value of None has not sense.
 268     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 269         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 270         insertions must take into account the deleted items.
 271     :param dict_to_change:  Target dictionary to be changed.
 272     :param dict_reference: Dictionary that contains changes to be applied.
 273     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 274     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 275     """
 276     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 277         to_append = {}
 278         to_insert_at_index = {}
 279         values_to_edit_delete = {}
 280         indexes_to_edit_delete = []
 281         array_edition = None
 282         _key_list.append("")
 283         for k in _dict_reference:
 284             _key_list[-1] = str(k)
 285             if not isinstance(k, str) or not k.startswith("$"):
 286                 if array_edition is True:
 287                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 288                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 289                 array_edition = False
 290                 continue
 291             else:
 292                 if array_edition is False:
 293                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 294                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 295                 array_edition = True
 296             insert = False
 297             indexes = []  # indexes to edit or insert
 298             kitem = k[1:]
 299             if kitem.startswith('+'):
 300                 insert = True
 301                 kitem = kitem[1:]
 302                 if _dict_reference[k] is None:
 303                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 304                         ":".join(_key_list)))
 305
 306             if kitem.startswith('[') and kitem.endswith(']'):
 307                 try:
 308                     index = int(kitem[1:-1])
 309                     if index < 0:
 310                         index += len(array_to_change)
 311                     if index < 0:
 312                         index = 0  # skip outside index edition
 313                     indexes.append(index)
 314                 except Exception:
 315                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 316                         ":".join(_key_list)))
 317             elif kitem:
 318                 # match_found_skip = False
 319                 try:
 320                     filter_in = yaml.safe_load(kitem)
 321                 except Exception:
 322                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 323                 if isinstance(filter_in, dict):
 324                     for index, item in enumerate(array_to_change):
 325                         for filter_k, filter_v in filter_in.items():
 326                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 327                                 break
 328                         else:  # match found
 329                             if insert:
 330                                 # match_found_skip = True
 331                                 insert = False
 332                                 break
 333                             else:
 334                                 indexes.append(index)
 335                 else:
 336                     index = 0
 337                     try:
 338                         while True:  # if not match a ValueError exception will be raise
 339                             index = array_to_change.index(filter_in, index)
 340                             if insert:
 341                                 # match_found_skip = True
 342                                 insert = False
 343                                 break
 344                             indexes.append(index)
 345                             index += 1
 346                     except ValueError:
 347                         pass
 348
 349                 # if match_found_skip:
 350                 #     continue
 351             elif not insert:
 352                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 353                     ":".join(_key_list)))
 354             for index in indexes:
 355                 if insert:
 356                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 357                         # Several different insertions on the same item of the array
 358                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 359                             ":".join(_key_list), index))
 360                     to_insert_at_index[index] = _dict_reference[k]
 361                 else:
 362                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 363                         # Several different editions on the same item of the array
 364                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 365                             ":".join(_key_list), index))
 366                     indexes_to_edit_delete.append(index)
 367                     values_to_edit_delete[index] = _dict_reference[k]
 368             if not indexes:
 369                 if insert:
 370                     to_append[k] = _dict_reference[k]
 371                 # elif _dict_reference[k] is not None:
 372                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 373                 #         ":".join(_key_list)))
 374
 375         # edition/deletion is done before insertion
 376         indexes_to_edit_delete.sort(reverse=True)
 377         for index in indexes_to_edit_delete:
 378             _key_list[-1] = str(index)
 379             try:
 380                 if values_to_edit_delete[index] is None:  # None->Anything
 381                     try:
 382                         del (array_to_change[index])
 383                     except IndexError:
 384                         pass  # it is not consider an error if this index does not exist
 385                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 386                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 387                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 388                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 389                 else:  # Dict->NotDict
 390                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 391                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 392                             continue
 393                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 394                     # calling deep_update_rfc7396 to delete the None values
 395                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 396             except IndexError:
 397                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 398
 399         # insertion with indexes
 400         to_insert_indexes = list(to_insert_at_index.keys())
 401         to_insert_indexes.sort(reverse=True)
 402         for index in to_insert_indexes:
 403             array_to_change.insert(index, to_insert_at_index[index])
 404
 405         # append
 406         for k, insert_value in to_append.items():
 407             _key_list[-1] = str(k)
 408             insert_value_copy = deepcopy(insert_value)
 409             if isinstance(insert_value_copy, dict):
 410                 # calling deep_update_rfc7396 to delete the None values
 411                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 412             array_to_change.append(insert_value_copy)
 413
 414         _key_list.pop()
 415         if array_edition:
 416             return True
 417         return False
 418
 419     if key_list is None:
 420         key_list = []
 421     key_list.append("")
 422     for k in dict_reference:
 423         key_list[-1] = str(k)
 424         if dict_reference[k] is None:   # None->Anything
 425             if k in dict_to_change:
 426                 del dict_to_change[k]
 427         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 428             dict_to_change[k] = deepcopy(dict_reference[k])
 429         elif k not in dict_to_change:  # Dict->Empty
 430             dict_to_change[k] = deepcopy(dict_reference[k])
 431             # calling deep_update_rfc7396 to delete the None values
 432             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 433         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 434             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 435         else:       # Dict->NotDict
 436             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 437                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 438                     continue
 439             dict_to_change[k] = deepcopy(dict_reference[k])
 440             # calling deep_update_rfc7396 to delete the None values
 441             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 442     key_list.pop()
 443
 444
 445 def deep_update(dict_to_change, dict_reference):
 446     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 447     return deep_update_rfc7396(dict_to_change, dict_reference)