osm_common/dbbase.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import yaml
  19 import logging
  20 from http import HTTPStatus
  21 from copy import deepcopy
  22
  23 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  24
  25
  26 class DbException(Exception):
  27
  28     def __init__(self, message, http_code=HTTPStatus.NOT_FOUND):
  29         self.http_code = http_code
  30         Exception.__init__(self, "database exception " + str(message))
  31
  32
  33 class DbBase(object):
  34
  35     def __init__(self, logger_name='db', master_password=None):
  36         """
  37         Constructor od dbBase
  38         :param logger_name: logging name
  39         :param master_password: master password used for encrypt decrypt methods
  40         """
  41         self.logger = logging.getLogger(logger_name)
  42         self.master_password = master_password
  43
  44     def db_connect(self, config):
  45         """
  46         Connect to database
  47         :param config: Configuration of database
  48         :return: None or raises DbException on error
  49         """
  50         pass
  51
  52     def db_disconnect(self):
  53         """
  54         Disconnect from database
  55         :return: None
  56         """
  57         pass
  58
  59     def get_list(self, table, q_filter=None):
  60         """
  61         Obtain a list of entries matching q_filter
  62         :param table: collection or table
  63         :param q_filter: Filter
  64         :return: a list (can be empty) with the found entries. Raises DbException on error
  65         """
  66         raise DbException("Method 'get_list' not implemented")
  67
  68     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
  69         """
  70         Obtain one entry matching q_filter
  71         :param table: collection or table
  72         :param q_filter: Filter
  73         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
  74         it raises a DbException
  75         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
  76         that it raises a DbException
  77         :return: The requested element, or None
  78         """
  79         raise DbException("Method 'get_one' not implemented")
  80
  81     def del_list(self, table, q_filter=None):
  82         """
  83         Deletes all entries that match q_filter
  84         :param table: collection or table
  85         :param q_filter: Filter
  86         :return: Dict with the number of entries deleted
  87         """
  88         raise DbException("Method 'del_list' not implemented")
  89
  90     def del_one(self, table, q_filter=None, fail_on_empty=True):
  91         """
  92         Deletes one entry that matches q_filter
  93         :param table: collection or table
  94         :param q_filter: Filter
  95         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
  96         which case it raises a DbException
  97         :return: Dict with the number of entries deleted
  98         """
  99         raise DbException("Method 'del_one' not implemented")
 100
 101     def create(self, table, indata):
 102         """
 103         Add a new entry at database
 104         :param table: collection or table
 105         :param indata: content to be added
 106         :return: database id of the inserted element. Raises a DbException on error
 107         """
 108         raise DbException("Method 'create' not implemented")
 109
 110     def set_one(self, table, q_filter, update_dict, fail_on_empty=True):
 111         """
 112         Modifies an entry at database
 113         :param table: collection or table
 114         :param q_filter: Filter
 115         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 116         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 117         it raises a DbException
 118         :return: Dict with the number of entries modified. None if no matching is found.
 119         """
 120         raise DbException("Method 'set_one' not implemented")
 121
 122     def set_list(self, table, q_filter, update_dict):
 123         """
 124         Modifies al matching entries at database
 125         :param table: collection or table
 126         :param q_filter: Filter
 127         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 128         :return: Dict with the number of entries modified
 129         """
 130         raise DbException("Method 'set_list' not implemented")
 131
 132     def replace(self, table, _id, indata, fail_on_empty=True):
 133         """
 134         Replace the content of an entry
 135         :param table: collection or table
 136         :param _id: internal database id
 137         :param indata: content to replace
 138         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 139         it raises a DbException
 140         :return: Dict with the number of entries replaced
 141         """
 142         raise DbException("Method 'replace' not implemented")
 143
 144     def encrypt(self, value, salt=None):
 145         """
 146         Encrypt a value
 147         :param value: value to be encrypted
 148         :param salt: optional salt to be used
 149         :return: Encrypted content of value
 150         """
 151         # for the moment return same value. until all modules call this method
 152         return value
 153         # raise DbException("Method 'encrypt' not implemented")
 154
 155     def decrypt(self, value, salt=None):
 156         """
 157         Decrypt an encrypted value
 158         :param value: value to be decrypted
 159         :param salt: optional salt to be used
 160         :return: Plain content of value
 161         """
 162         # for the moment return same value. until all modules call this method
 163         return value
 164         # raise DbException("Method 'decrypt' not implemented")
 165
 166
 167 def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None):
 168     """
 169     Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
 170     Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete.
 171     It implements an extra feature that allows modifying an array. RFC7396 only allows replacing the entire array.
 172     For that, dict_reference should contains a dict with keys starting by "$" with the following meaning:
 173         $[index]    <index> is an integer for targeting a concrete index from dict_to_change array. If the value is None
 174                     the element of the array is deleted, otherwise it is edited.
 175         $+[index]   The value is inserted at this <index>. A value of None has not sense and an exception is raised.
 176         $+          The value is appended at the end. A value of None has not sense and an exception is raised.
 177         $val        It looks for all the items in the array dict_to_change equal to <val>. <val> is evaluated as yaml,
 178                     that is, numbers are taken as type int, true/false as boolean, etc. Use quotes to force string.
 179                     Nothing happens if no match is found. If the value is None the matched elements are deleted.
 180         $key: val   In case a dictionary is passed in yaml format, if looks for all items in the array dict_to_change
 181                     that are dictionaries and contains this <key> equal to <val>. Several keys can be used by yaml
 182                     format '{key: val, key: val, ...}'; and all of them mast match. Nothing happens if no match is
 183                     found. If value is None the matched items are deleted, otherwise they are edited.
 184         $+val       If no match if found (see '$val'), the value is appended to the array. If any match is found nothing
 185                     is changed. A value of None has not sense.
 186         $+key: val  If no match if found (see '$key: val'), the value is appended to the array. If any match is found
 187                     nothing is changed. A value of None has not sense.
 188     If there are several editions, insertions and deletions; editions and deletions are done first in reverse index
 189         order; then insertions also in reverse index order; and finally appends in any order. So indexes used at
 190         insertions must take into account the deleted items.
 191     :param dict_to_change:  Target dictionary to be changed.
 192     :param dict_reference: Dictionary that contains changes to be applied.
 193     :param key_list: This is used internally for recursive calls. Do not fill this parameter.
 194     :return: none or raises and exception only at array modification when there is a bad format or conflict.
 195     """
 196     def _deep_update_array(array_to_change, _dict_reference, _key_list):
 197         to_append = {}
 198         to_insert_at_index = {}
 199         values_to_edit_delete = {}
 200         indexes_to_edit_delete = []
 201         array_edition = None
 202         _key_list.append("")
 203         for k in _dict_reference:
 204             _key_list[-1] = str(k)
 205             if not isinstance(k, str) or not k.startswith("$"):
 206                 if array_edition is True:
 207                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 208                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 209                 array_edition = False
 210                 continue
 211             else:
 212                 if array_edition is False:
 213                     raise DbException("Found array edition (keys starting with '$') and pure dictionary edition in the"
 214                                       " same dict at '{}'".format(":".join(_key_list[:-1])))
 215                 array_edition = True
 216             insert = False
 217             indexes = []  # indexes to edit or insert
 218             kitem = k[1:]
 219             if kitem.startswith('+'):
 220                 insert = True
 221                 kitem = kitem[1:]
 222                 if _dict_reference[k] is None:
 223                     raise DbException("A value of None has not sense for insertions at '{}'".format(
 224                         ":".join(_key_list)))
 225
 226             if kitem.startswith('[') and kitem.endswith(']'):
 227                 try:
 228                     index = int(kitem[1:-1])
 229                     if index < 0:
 230                         index += len(array_to_change)
 231                     if index < 0:
 232                         index = 0  # skip outside index edition
 233                     indexes.append(index)
 234                 except Exception:
 235                     raise DbException("Wrong format at '{}'. Expecting integer index inside quotes".format(
 236                         ":".join(_key_list)))
 237             elif kitem:
 238                 # match_found_skip = False
 239                 try:
 240                     filter_in = yaml.safe_load(kitem)
 241                 except Exception:
 242                     raise DbException("Wrong format at '{}'. Expecting '$<yaml-format>'".format(":".join(_key_list)))
 243                 if isinstance(filter_in, dict):
 244                     for index, item in enumerate(array_to_change):
 245                         for filter_k, filter_v in filter_in.items():
 246                             if not isinstance(item, dict) or filter_k not in item or item[filter_k] != filter_v:
 247                                 break
 248                         else:  # match found
 249                             if insert:
 250                                 # match_found_skip = True
 251                                 insert = False
 252                                 break
 253                             else:
 254                                 indexes.append(index)
 255                 else:
 256                     index = 0
 257                     try:
 258                         while True:  # if not match a ValueError exception will be raise
 259                             index = array_to_change.index(filter_in, index)
 260                             if insert:
 261                                 # match_found_skip = True
 262                                 insert = False
 263                                 break
 264                             indexes.append(index)
 265                             index += 1
 266                     except ValueError:
 267                         pass
 268
 269                 # if match_found_skip:
 270                 #     continue
 271             elif not insert:
 272                 raise DbException("Wrong format at '{}'. Expecting '$+', '$[<index]' or '$[<filter>]'".format(
 273                     ":".join(_key_list)))
 274             for index in indexes:
 275                 if insert:
 276                     if index in to_insert_at_index and to_insert_at_index[index] != _dict_reference[k]:
 277                         # Several different insertions on the same item of the array
 278                         raise DbException("Conflict at '{}'. Several insertions on same array index {}".format(
 279                             ":".join(_key_list), index))
 280                     to_insert_at_index[index] = _dict_reference[k]
 281                 else:
 282                     if index in indexes_to_edit_delete and values_to_edit_delete[index] != _dict_reference[k]:
 283                         # Several different editions on the same item of the array
 284                         raise DbException("Conflict at '{}'. Several editions on array index {}".format(
 285                             ":".join(_key_list), index))
 286                     indexes_to_edit_delete.append(index)
 287                     values_to_edit_delete[index] = _dict_reference[k]
 288             if not indexes:
 289                 if insert:
 290                     to_append[k] = _dict_reference[k]
 291                 # elif _dict_reference[k] is not None:
 292                 #     raise DbException("Not found any match to edit in the array, or wrong format at '{}'".format(
 293                 #         ":".join(_key_list)))
 294
 295         # edition/deletion is done before insertion
 296         indexes_to_edit_delete.sort(reverse=True)
 297         for index in indexes_to_edit_delete:
 298             _key_list[-1] = str(index)
 299             try:
 300                 if values_to_edit_delete[index] is None:  # None->Anything
 301                     try:
 302                         del (array_to_change[index])
 303                     except IndexError:
 304                         pass  # it is not consider an error if this index does not exist
 305                 elif not isinstance(values_to_edit_delete[index], dict):  # NotDict->Anything
 306                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 307                 elif isinstance(array_to_change[index], dict):  # Dict->Dict
 308                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 309                 else:  # Dict->NotDict
 310                     if isinstance(array_to_change[index], list):  # Dict->List. Check extra array edition
 311                         if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list):
 312                             continue
 313                     array_to_change[index] = deepcopy(values_to_edit_delete[index])
 314                     # calling deep_update_rfc7396 to delete the None values
 315                     deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list)
 316             except IndexError:
 317                 raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list)))
 318
 319         # insertion with indexes
 320         to_insert_indexes = list(to_insert_at_index.keys())
 321         to_insert_indexes.sort(reverse=True)
 322         for index in to_insert_indexes:
 323             array_to_change.insert(index, to_insert_at_index[index])
 324
 325         # append
 326         for k, insert_value in to_append.items():
 327             _key_list[-1] = str(k)
 328             insert_value_copy = deepcopy(insert_value)
 329             if isinstance(insert_value_copy, dict):
 330                 # calling deep_update_rfc7396 to delete the None values
 331                 deep_update_rfc7396(insert_value_copy, insert_value, _key_list)
 332             array_to_change.append(insert_value_copy)
 333
 334         _key_list.pop()
 335         if array_edition:
 336             return True
 337         return False
 338
 339     if key_list is None:
 340         key_list = []
 341     key_list.append("")
 342     for k in dict_reference:
 343         key_list[-1] = str(k)
 344         if dict_reference[k] is None:   # None->Anything
 345             if k in dict_to_change:
 346                 del dict_to_change[k]
 347         elif not isinstance(dict_reference[k], dict):  # NotDict->Anything
 348             dict_to_change[k] = deepcopy(dict_reference[k])
 349         elif k not in dict_to_change:  # Dict->Empty
 350             dict_to_change[k] = deepcopy(dict_reference[k])
 351             # calling deep_update_rfc7396 to delete the None values
 352             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 353         elif isinstance(dict_to_change[k], dict):  # Dict->Dict
 354             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 355         else:       # Dict->NotDict
 356             if isinstance(dict_to_change[k], list):  # Dict->List. Check extra array edition
 357                 if _deep_update_array(dict_to_change[k], dict_reference[k], key_list):
 358                     continue
 359             dict_to_change[k] = deepcopy(dict_reference[k])
 360             # calling deep_update_rfc7396 to delete the None values
 361             deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list)
 362     key_list.pop()
 363
 364
 365 def deep_update(dict_to_change, dict_reference):
 366     """ Maintained for backward compatibility. Use deep_update_rfc7396 instead"""
 367     return deep_update_rfc7396(dict_to_change, dict_reference)