osm_common/dbmemory.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import logging
  19 from osm_common.dbbase import DbException, DbBase
  20 from osm_common.dbmongo import deep_update
  21 from http import HTTPStatus
  22 from uuid import uuid4
  23 from copy import deepcopy
  24
  25 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  26
  27
  28 class DbMemory(DbBase):
  29
  30     def __init__(self, logger_name='db', lock=False):
  31         super().__init__(logger_name, lock)
  32         self.db = {}
  33
  34     def db_connect(self, config):
  35         """
  36         Connect to database
  37         :param config: Configuration of database
  38         :return: None or raises DbException on error
  39         """
  40         if "logger_name" in config:
  41             self.logger = logging.getLogger(config["logger_name"])
  42         master_key = config.get("commonkey") or config.get("masterpassword")
  43         if master_key:
  44             self.set_secret_key(master_key)
  45
  46     @staticmethod
  47     def _format_filter(q_filter):
  48         db_filter = {}
  49         # split keys with ANYINDEX in this way:
  50         # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v }  -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
  51         if q_filter:
  52             for k, v in q_filter.items():
  53                 db_v = v
  54                 kleft, _, kright = k.rpartition(".ANYINDEX.")
  55                 while kleft:
  56                     k = kleft + ".ANYINDEX"
  57                     db_v = {kright: db_v}
  58                     kleft, _, kright = k.rpartition(".ANYINDEX.")
  59                 deep_update(db_filter, {k: db_v})
  60
  61         return db_filter
  62
  63     def _find(self, table, q_filter):
  64
  65         def recursive_find(key_list, key_next_index, content, operator, target):
  66             if key_next_index == len(key_list) or content is None:
  67                 try:
  68                     if operator == "eq":
  69                         if isinstance(target, list) and not isinstance(content, list):
  70                             return True if content in target else False
  71                         return True if content == target else False
  72                     elif operator in ("neq", "ne"):
  73                         if isinstance(target, list) and not isinstance(content, list):
  74                             return True if content not in target else False
  75                         return True if content != target else False
  76                     if operator == "gt":
  77                         return content > target
  78                     elif operator == "gte":
  79                         return content >= target
  80                     elif operator == "lt":
  81                         return content < target
  82                     elif operator == "lte":
  83                         return content <= target
  84                     elif operator == "cont":
  85                         return content in target
  86                     elif operator == "ncont":
  87                         return content not in target
  88                     else:
  89                         raise DbException("Unknown filter operator '{}' in key '{}'".
  90                                           format(operator, ".".join(key_list)), http_code=HTTPStatus.BAD_REQUEST)
  91                 except TypeError:
  92                     return False
  93
  94             elif isinstance(content, dict):
  95                 return recursive_find(key_list, key_next_index+1, content.get(key_list[key_next_index]), operator,
  96                                       target)
  97             elif isinstance(content, list):
  98                 look_for_match = True  # when there is a match return immediately
  99                 if (target is None and operator not in ("neq", "ne")) or \
 100                         (target is not None and operator in ("neq", "ne")):
 101                     look_for_match = False  # when there is a match return immediately
 102
 103                 for content_item in content:
 104                     if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict):
 105                         for k2, v2 in target.items():
 106                             k_new_list = k2.split(".")
 107                             new_operator = "eq"
 108                             if k_new_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
 109                                 new_operator = k_new_list.pop()
 110                             if not recursive_find(k_new_list, 0, content_item, new_operator, v2):
 111                                 match = False
 112                                 break
 113                         else:
 114                             match = True
 115
 116                     else:
 117                         match = recursive_find(key_list, key_next_index, content_item, operator, target)
 118                     if match == look_for_match:
 119                         return match
 120                 if key_list[key_next_index].isdecimal() and int(key_list[key_next_index]) < len(content):
 121                     match = recursive_find(key_list, key_next_index+1, content[int(key_list[key_next_index])],
 122                                            operator, target)
 123                     if match == look_for_match:
 124                         return match
 125                 return not look_for_match
 126             else:  # content is not dict, nor list neither None, so not found
 127                 if operator in ("neq", "ne"):
 128                     return True if target is None else False
 129                 else:
 130                     return True if target is None else False
 131
 132         for i, row in enumerate(self.db.get(table, ())):
 133             q_filter = q_filter or {}
 134             for k, v in q_filter.items():
 135                 k_list = k.split(".")
 136                 operator = "eq"
 137                 if k_list[-1] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", "ncont", "neq"):
 138                     operator = k_list.pop()
 139                 match = recursive_find(k_list, 0, row, operator, v)
 140                 if not match:
 141                     break
 142             else:
 143                 # match
 144                 yield i, row
 145
 146     def get_list(self, table, q_filter=None):
 147         """
 148         Obtain a list of entries matching q_filter
 149         :param table: collection or table
 150         :param q_filter: Filter
 151         :return: a list (can be empty) with the found entries. Raises DbException on error
 152         """
 153         try:
 154             result = []
 155             with self.lock:
 156                 for _, row in self._find(table, self._format_filter(q_filter)):
 157                     result.append(deepcopy(row))
 158             return result
 159         except DbException:
 160             raise
 161         except Exception as e:  # TODO refine
 162             raise DbException(str(e))
 163
 164     def count(self, table, q_filter=None):
 165         """
 166         Count the number of entries matching q_filter
 167         :param table: collection or table
 168         :param q_filter: Filter
 169         :return: number of entries found (can be zero)
 170         :raise: DbException on error
 171         """
 172         try:
 173             with self.lock:
 174                 return sum(1 for x in self._find(table, self._format_filter(q_filter)))
 175         except DbException:
 176             raise
 177         except Exception as e:  # TODO refine
 178             raise DbException(str(e))
 179
 180     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 181         """
 182         Obtain one entry matching q_filter
 183         :param table: collection or table
 184         :param q_filter: Filter
 185         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 186         it raises a DbException
 187         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 188         that it raises a DbException
 189         :return: The requested element, or None
 190         """
 191         try:
 192             result = None
 193             with self.lock:
 194                 for _, row in self._find(table, self._format_filter(q_filter)):
 195                     if not fail_on_more:
 196                         return deepcopy(row)
 197                     if result:
 198                         raise DbException("Found more than one entry with filter='{}'".format(q_filter),
 199                                           HTTPStatus.CONFLICT.value)
 200                     result = row
 201             if not result and fail_on_empty:
 202                 raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND)
 203             return deepcopy(result)
 204         except Exception as e:  # TODO refine
 205             raise DbException(str(e))
 206
 207     def del_list(self, table, q_filter=None):
 208         """
 209         Deletes all entries that match q_filter
 210         :param table: collection or table
 211         :param q_filter: Filter
 212         :return: Dict with the number of entries deleted
 213         """
 214         try:
 215             id_list = []
 216             with self.lock:
 217                 for i, _ in self._find(table, self._format_filter(q_filter)):
 218                     id_list.append(i)
 219             deleted = len(id_list)
 220             for i in reversed(id_list):
 221                 del self.db[table][i]
 222             return {"deleted": deleted}
 223         except DbException:
 224             raise
 225         except Exception as e:  # TODO refine
 226             raise DbException(str(e))
 227
 228     def del_one(self, table, q_filter=None, fail_on_empty=True):
 229         """
 230         Deletes one entry that matches q_filter
 231         :param table: collection or table
 232         :param q_filter: Filter
 233         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 234         which case it raises a DbException
 235         :return: Dict with the number of entries deleted
 236         """
 237         try:
 238             with self.lock:
 239                 for i, _ in self._find(table, self._format_filter(q_filter)):
 240                     break
 241                 else:
 242                     if fail_on_empty:
 243                         raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND)
 244                     return None
 245                 del self.db[table][i]
 246             return {"deleted": 1}
 247         except Exception as e:  # TODO refine
 248             raise DbException(str(e))
 249
 250     def set_one(self, table, q_filter, update_dict, fail_on_empty=True, unset=None, pull=None, push=None):
 251         """
 252         Modifies an entry at database
 253         :param table: collection or table
 254         :param q_filter: Filter
 255         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 256         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 257         it raises a DbException
 258         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 259                       ignored. If not exist, it is ignored
 260         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 261                      if exist in the array is removed. If not exist, it is ignored
 262         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 263                      is appended to the end of the array
 264         :return: Dict with the number of entries modified. None if no matching is found.
 265         """
 266         try:
 267             with self.lock:
 268                 for i, db_item in self._find(table, self._format_filter(q_filter)):
 269                     break
 270                 else:
 271                     if fail_on_empty:
 272                         raise DbException("Not found entry with _id='{}'".format(q_filter), HTTPStatus.NOT_FOUND)
 273                     return None
 274                 for k, v in update_dict.items():
 275                     db_nested = db_item
 276                     k_list = k.split(".")
 277                     k_nested_prev = k_list[0]
 278                     for k_nested in k_list[1:]:
 279                         if isinstance(db_nested[k_nested_prev], dict):
 280                             if k_nested not in db_nested[k_nested_prev]:
 281                                 db_nested[k_nested_prev][k_nested] = None
 282                         elif isinstance(db_nested[k_nested_prev], list) and k_nested.isdigit():
 283                             # extend list with Nones if index greater than list
 284                             k_nested = int(k_nested)
 285                             if k_nested >= len(db_nested[k_nested_prev]):
 286                                 db_nested[k_nested_prev] += [None] * (k_nested - len(db_nested[k_nested_prev]) + 1)
 287                         elif db_nested[k_nested_prev] is None:
 288                             db_nested[k_nested_prev] = {k_nested: None}
 289                         else:  # number, string, boolean, ... or list but with not integer key
 290                             raise DbException("Cannot set '{}' on existing '{}={}'".format(k, k_nested_prev,
 291                                                                                            db_nested[k_nested_prev]))
 292
 293                         db_nested = db_nested[k_nested_prev]
 294                         k_nested_prev = k_nested
 295
 296                     db_nested[k_nested_prev] = v
 297                 return {"updated": 1}
 298         except DbException:
 299             raise
 300         except Exception as e:  # TODO refine
 301             raise DbException(str(e))
 302
 303     def replace(self, table, _id, indata, fail_on_empty=True):
 304         """
 305         Replace the content of an entry
 306         :param table: collection or table
 307         :param _id: internal database id
 308         :param indata: content to replace
 309         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 310         it raises a DbException
 311         :return: Dict with the number of entries replaced
 312         """
 313         try:
 314             with self.lock:
 315                 for i, _ in self._find(table, self._format_filter({"_id": _id})):
 316                     break
 317                 else:
 318                     if fail_on_empty:
 319                         raise DbException("Not found entry with _id='{}'".format(_id), HTTPStatus.NOT_FOUND)
 320                     return None
 321                 self.db[table][i] = deepcopy(indata)
 322             return {"updated": 1}
 323         except DbException:
 324             raise
 325         except Exception as e:  # TODO refine
 326             raise DbException(str(e))
 327
 328     def create(self, table, indata):
 329         """
 330         Add a new entry at database
 331         :param table: collection or table
 332         :param indata: content to be added
 333         :return: database id of the inserted element. Raises a DbException on error
 334         """
 335         try:
 336             id = indata.get("_id")
 337             if not id:
 338                 id = str(uuid4())
 339                 indata["_id"] = id
 340             with self.lock:
 341                 if table not in self.db:
 342                     self.db[table] = []
 343                 self.db[table].append(deepcopy(indata))
 344             return id
 345         except Exception as e:  # TODO refine
 346             raise DbException(str(e))
 347
 348     def create_list(self, table, indata_list):
 349         """
 350         Add a new entry at database
 351         :param table: collection or table
 352         :param indata_list: list content to be added
 353         :return: database ids of the inserted element. Raises a DbException on error
 354         """
 355         try:
 356             _ids = []
 357             for indata in indata_list:
 358                 _id = indata.get("_id")
 359                 if not _id:
 360                     _id = str(uuid4())
 361                     indata["_id"] = _id
 362                 with self.lock:
 363                     if table not in self.db:
 364                         self.db[table] = []
 365                     self.db[table].append(deepcopy(indata))
 366                 _ids.append(_id)
 367             return _ids
 368         except Exception as e:  # TODO refine
 369             raise DbException(str(e))
 370
 371
 372 if __name__ == '__main__':
 373     # some test code
 374     db = DbMemory()
 375     db.create("test", {"_id": 1, "data": 1})
 376     db.create("test", {"_id": 2, "data": 2})
 377     db.create("test", {"_id": 3, "data": 3})
 378     print("must be 3 items:", db.get_list("test"))
 379     print("must return item 2:", db.get_list("test", {"_id": 2}))
 380     db.del_one("test", {"_id": 2})
 381     print("must be emtpy:", db.get_list("test", {"_id": 2}))