osm_common/dbmemory.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 import logging
  19 from osm_common.dbbase import DbException, DbBase
  20 from osm_common.dbmongo import deep_update
  21 from http import HTTPStatus
  22 from uuid import uuid4
  23 from copy import deepcopy
  24
  25 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  26
  27
  28 class DbMemory(DbBase):
  29     def __init__(self, logger_name="db", lock=False):
  30         super().__init__(logger_name, lock)
  31         self.db = {}
  32
  33     def db_connect(self, config):
  34         """
  35         Connect to database
  36         :param config: Configuration of database
  37         :return: None or raises DbException on error
  38         """
  39         if "logger_name" in config:
  40             self.logger = logging.getLogger(config["logger_name"])
  41         master_key = config.get("commonkey") or config.get("masterpassword")
  42         if master_key:
  43             self.set_secret_key(master_key)
  44
  45     @staticmethod
  46     def _format_filter(q_filter):
  47         db_filter = {}
  48         # split keys with ANYINDEX in this way:
  49         # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v }  -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
  50         if q_filter:
  51             for k, v in q_filter.items():
  52                 db_v = v
  53                 kleft, _, kright = k.rpartition(".ANYINDEX.")
  54                 while kleft:
  55                     k = kleft + ".ANYINDEX"
  56                     db_v = {kright: db_v}
  57                     kleft, _, kright = k.rpartition(".ANYINDEX.")
  58                 deep_update(db_filter, {k: db_v})
  59
  60         return db_filter
  61
  62     def _find(self, table, q_filter):
  63         def recursive_find(key_list, key_next_index, content, oper, target):
  64             if key_next_index == len(key_list) or content is None:
  65                 try:
  66                     if oper in ("eq", "cont"):
  67                         if isinstance(target, list):
  68                             if isinstance(content, list):
  69                                 return any(
  70                                     content_item in target for content_item in content
  71                                 )
  72                             return content in target
  73                         elif isinstance(content, list):
  74                             return target in content
  75                         else:
  76                             return content == target
  77                     elif oper in ("neq", "ne", "ncont"):
  78                         if isinstance(target, list):
  79                             if isinstance(content, list):
  80                                 return all(
  81                                     content_item not in target
  82                                     for content_item in content
  83                                 )
  84                             return content not in target
  85                         elif isinstance(content, list):
  86                             return target not in content
  87                         else:
  88                             return content != target
  89                     if oper == "gt":
  90                         return content > target
  91                     elif oper == "gte":
  92                         return content >= target
  93                     elif oper == "lt":
  94                         return content < target
  95                     elif oper == "lte":
  96                         return content <= target
  97                     else:
  98                         raise DbException(
  99                             "Unknown filter operator '{}' in key '{}'".format(
 100                                 oper, ".".join(key_list)
 101                             ),
 102                             http_code=HTTPStatus.BAD_REQUEST,
 103                         )
 104                 except TypeError:
 105                     return False
 106
 107             elif isinstance(content, dict):
 108                 return recursive_find(
 109                     key_list,
 110                     key_next_index + 1,
 111                     content.get(key_list[key_next_index]),
 112                     oper,
 113                     target,
 114                 )
 115             elif isinstance(content, list):
 116                 look_for_match = True  # when there is a match return immediately
 117                 if (target is None) != (
 118                     oper in ("neq", "ne", "ncont")
 119                 ):  # one True and other False (Xor)
 120                     look_for_match = (
 121                         False  # when there is not a match return immediately
 122                     )
 123
 124                 for content_item in content:
 125                     if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict):
 126                         matches = True
 127                         for k2, v2 in target.items():
 128                             k_new_list = k2.split(".")
 129                             new_operator = "eq"
 130                             if k_new_list[-1] in (
 131                                 "eq",
 132                                 "ne",
 133                                 "gt",
 134                                 "gte",
 135                                 "lt",
 136                                 "lte",
 137                                 "cont",
 138                                 "ncont",
 139                                 "neq",
 140                             ):
 141                                 new_operator = k_new_list.pop()
 142                             if not recursive_find(
 143                                 k_new_list, 0, content_item, new_operator, v2
 144                             ):
 145                                 matches = False
 146                                 break
 147
 148                     else:
 149                         matches = recursive_find(
 150                             key_list, key_next_index, content_item, oper, target
 151                         )
 152                     if matches == look_for_match:
 153                         return matches
 154                 if key_list[key_next_index].isdecimal() and int(
 155                     key_list[key_next_index]
 156                 ) < len(content):
 157                     matches = recursive_find(
 158                         key_list,
 159                         key_next_index + 1,
 160                         content[int(key_list[key_next_index])],
 161                         oper,
 162                         target,
 163                     )
 164                     if matches == look_for_match:
 165                         return matches
 166                 return not look_for_match
 167             else:  # content is not dict, nor list neither None, so not found
 168                 if oper in ("neq", "ne", "ncont"):
 169                     return target is not None
 170                 else:
 171                     return target is None
 172
 173         for i, row in enumerate(self.db.get(table, ())):
 174             q_filter = q_filter or {}
 175             for k, v in q_filter.items():
 176                 k_list = k.split(".")
 177                 operator = "eq"
 178                 if k_list[-1] in (
 179                     "eq",
 180                     "ne",
 181                     "gt",
 182                     "gte",
 183                     "lt",
 184                     "lte",
 185                     "cont",
 186                     "ncont",
 187                     "neq",
 188                 ):
 189                     operator = k_list.pop()
 190                 matches = recursive_find(k_list, 0, row, operator, v)
 191                 if not matches:
 192                     break
 193             else:
 194                 # match
 195                 yield i, row
 196
 197     def get_list(self, table, q_filter=None):
 198         """
 199         Obtain a list of entries matching q_filter
 200         :param table: collection or table
 201         :param q_filter: Filter
 202         :return: a list (can be empty) with the found entries. Raises DbException on error
 203         """
 204         try:
 205             result = []
 206             with self.lock:
 207                 for _, row in self._find(table, self._format_filter(q_filter)):
 208                     result.append(deepcopy(row))
 209             return result
 210         except DbException:
 211             raise
 212         except Exception as e:  # TODO refine
 213             raise DbException(str(e))
 214
 215     def count(self, table, q_filter=None):
 216         """
 217         Count the number of entries matching q_filter
 218         :param table: collection or table
 219         :param q_filter: Filter
 220         :return: number of entries found (can be zero)
 221         :raise: DbException on error
 222         """
 223         try:
 224             with self.lock:
 225                 return sum(1 for x in self._find(table, self._format_filter(q_filter)))
 226         except DbException:
 227             raise
 228         except Exception as e:  # TODO refine
 229             raise DbException(str(e))
 230
 231     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 232         """
 233         Obtain one entry matching q_filter
 234         :param table: collection or table
 235         :param q_filter: Filter
 236         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 237         it raises a DbException
 238         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 239         that it raises a DbException
 240         :return: The requested element, or None
 241         """
 242         try:
 243             result = None
 244             with self.lock:
 245                 for _, row in self._find(table, self._format_filter(q_filter)):
 246                     if not fail_on_more:
 247                         return deepcopy(row)
 248                     if result:
 249                         raise DbException(
 250                             "Found more than one entry with filter='{}'".format(
 251                                 q_filter
 252                             ),
 253                             HTTPStatus.CONFLICT.value,
 254                         )
 255                     result = row
 256             if not result and fail_on_empty:
 257                 raise DbException(
 258                     "Not found entry with filter='{}'".format(q_filter),
 259                     HTTPStatus.NOT_FOUND,
 260                 )
 261             return deepcopy(result)
 262         except Exception as e:  # TODO refine
 263             raise DbException(str(e))
 264
 265     def del_list(self, table, q_filter=None):
 266         """
 267         Deletes all entries that match q_filter
 268         :param table: collection or table
 269         :param q_filter: Filter
 270         :return: Dict with the number of entries deleted
 271         """
 272         try:
 273             id_list = []
 274             with self.lock:
 275                 for i, _ in self._find(table, self._format_filter(q_filter)):
 276                     id_list.append(i)
 277             deleted = len(id_list)
 278             for i in reversed(id_list):
 279                 del self.db[table][i]
 280             return {"deleted": deleted}
 281         except DbException:
 282             raise
 283         except Exception as e:  # TODO refine
 284             raise DbException(str(e))
 285
 286     def del_one(self, table, q_filter=None, fail_on_empty=True):
 287         """
 288         Deletes one entry that matches q_filter
 289         :param table: collection or table
 290         :param q_filter: Filter
 291         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 292         which case it raises a DbException
 293         :return: Dict with the number of entries deleted
 294         """
 295         try:
 296             with self.lock:
 297                 for i, _ in self._find(table, self._format_filter(q_filter)):
 298                     break
 299                 else:
 300                     if fail_on_empty:
 301                         raise DbException(
 302                             "Not found entry with filter='{}'".format(q_filter),
 303                             HTTPStatus.NOT_FOUND,
 304                         )
 305                     return None
 306                 del self.db[table][i]
 307             return {"deleted": 1}
 308         except Exception as e:  # TODO refine
 309             raise DbException(str(e))
 310
 311     def _update(
 312         self,
 313         db_item,
 314         update_dict,
 315         unset=None,
 316         pull=None,
 317         push=None,
 318         push_list=None,
 319         pull_list=None,
 320     ):
 321         """
 322         Modifies an entry at database
 323         :param db_item: entry of the table to update
 324         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 325         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 326                       ignored. If not exist, it is ignored
 327         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 328                      if exist in the array is removed. If not exist, it is ignored
 329         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 330         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 331                      is appended to the end of the array
 332         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 333                           whole array
 334         :return: True if database has been changed, False if not; Exception on error
 335         """
 336
 337         def _iterate_keys(k, db_nested, populate=True):
 338             k_list = k.split(".")
 339             k_item_prev = k_list[0]
 340             populated = False
 341             if k_item_prev not in db_nested and populate:
 342                 populated = True
 343                 db_nested[k_item_prev] = None
 344             for k_item in k_list[1:]:
 345                 if isinstance(db_nested[k_item_prev], dict):
 346                     if k_item not in db_nested[k_item_prev]:
 347                         if not populate:
 348                             raise DbException(
 349                                 "Cannot set '{}', not existing '{}'".format(k, k_item)
 350                             )
 351                         populated = True
 352                         db_nested[k_item_prev][k_item] = None
 353                 elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit():
 354                     # extend list with Nones if index greater than list
 355                     k_item = int(k_item)
 356                     if k_item >= len(db_nested[k_item_prev]):
 357                         if not populate:
 358                             raise DbException(
 359                                 "Cannot set '{}', index too large '{}'".format(
 360                                     k, k_item
 361                                 )
 362                             )
 363                         populated = True
 364                         db_nested[k_item_prev] += [None] * (
 365                             k_item - len(db_nested[k_item_prev]) + 1
 366                         )
 367                 elif db_nested[k_item_prev] is None:
 368                     if not populate:
 369                         raise DbException(
 370                             "Cannot set '{}', not existing '{}'".format(k, k_item)
 371                         )
 372                     populated = True
 373                     db_nested[k_item_prev] = {k_item: None}
 374                 else:  # number, string, boolean, ... or list but with not integer key
 375                     raise DbException(
 376                         "Cannot set '{}' on existing '{}={}'".format(
 377                             k, k_item_prev, db_nested[k_item_prev]
 378                         )
 379                     )
 380                 db_nested = db_nested[k_item_prev]
 381                 k_item_prev = k_item
 382             return db_nested, k_item_prev, populated
 383
 384         updated = False
 385         try:
 386             if update_dict:
 387                 for dot_k, v in update_dict.items():
 388                     dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item)
 389                     dict_to_update[key_to_update] = v
 390                     updated = True
 391             if unset:
 392                 for dot_k in unset:
 393                     try:
 394                         dict_to_update, key_to_update, _ = _iterate_keys(
 395                             dot_k, db_item, populate=False
 396                         )
 397                         del dict_to_update[key_to_update]
 398                         updated = True
 399                     except Exception:
 400                         pass
 401             if pull:
 402                 for dot_k, v in pull.items():
 403                     try:
 404                         dict_to_update, key_to_update, _ = _iterate_keys(
 405                             dot_k, db_item, populate=False
 406                         )
 407                     except Exception:
 408                         continue
 409                     if key_to_update not in dict_to_update:
 410                         continue
 411                     if not isinstance(dict_to_update[key_to_update], list):
 412                         raise DbException(
 413                             "Cannot pull '{}'. Target is not a list".format(dot_k)
 414                         )
 415                     while v in dict_to_update[key_to_update]:
 416                         dict_to_update[key_to_update].remove(v)
 417                         updated = True
 418             if pull_list:
 419                 for dot_k, v in pull_list.items():
 420                     if not isinstance(v, list):
 421                         raise DbException(
 422                             "Invalid content at pull_list, '{}' must be an array".format(
 423                                 dot_k
 424                             ),
 425                             http_code=HTTPStatus.BAD_REQUEST,
 426                         )
 427                     try:
 428                         dict_to_update, key_to_update, _ = _iterate_keys(
 429                             dot_k, db_item, populate=False
 430                         )
 431                     except Exception:
 432                         continue
 433                     if key_to_update not in dict_to_update:
 434                         continue
 435                     if not isinstance(dict_to_update[key_to_update], list):
 436                         raise DbException(
 437                             "Cannot pull_list '{}'. Target is not a list".format(dot_k)
 438                         )
 439                     for single_v in v:
 440                         while single_v in dict_to_update[key_to_update]:
 441                             dict_to_update[key_to_update].remove(single_v)
 442                             updated = True
 443             if push:
 444                 for dot_k, v in push.items():
 445                     dict_to_update, key_to_update, populated = _iterate_keys(
 446                         dot_k, db_item
 447                     )
 448                     if (
 449                         isinstance(dict_to_update, dict)
 450                         and key_to_update not in dict_to_update
 451                     ):
 452                         dict_to_update[key_to_update] = [v]
 453                         updated = True
 454                     elif populated and dict_to_update[key_to_update] is None:
 455                         dict_to_update[key_to_update] = [v]
 456                         updated = True
 457                     elif not isinstance(dict_to_update[key_to_update], list):
 458                         raise DbException(
 459                             "Cannot push '{}'. Target is not a list".format(dot_k)
 460                         )
 461                     else:
 462                         dict_to_update[key_to_update].append(v)
 463                         updated = True
 464             if push_list:
 465                 for dot_k, v in push_list.items():
 466                     if not isinstance(v, list):
 467                         raise DbException(
 468                             "Invalid content at push_list, '{}' must be an array".format(
 469                                 dot_k
 470                             ),
 471                             http_code=HTTPStatus.BAD_REQUEST,
 472                         )
 473                     dict_to_update, key_to_update, populated = _iterate_keys(
 474                         dot_k, db_item
 475                     )
 476                     if (
 477                         isinstance(dict_to_update, dict)
 478                         and key_to_update not in dict_to_update
 479                     ):
 480                         dict_to_update[key_to_update] = v.copy()
 481                         updated = True
 482                     elif populated and dict_to_update[key_to_update] is None:
 483                         dict_to_update[key_to_update] = v.copy()
 484                         updated = True
 485                     elif not isinstance(dict_to_update[key_to_update], list):
 486                         raise DbException(
 487                             "Cannot push '{}'. Target is not a list".format(dot_k),
 488                             http_code=HTTPStatus.CONFLICT,
 489                         )
 490                     else:
 491                         dict_to_update[key_to_update] += v
 492                         updated = True
 493
 494             return updated
 495         except DbException:
 496             raise
 497         except Exception as e:  # TODO refine
 498             raise DbException(str(e))
 499
 500     def set_one(
 501         self,
 502         table,
 503         q_filter,
 504         update_dict,
 505         fail_on_empty=True,
 506         unset=None,
 507         pull=None,
 508         push=None,
 509         push_list=None,
 510         pull_list=None,
 511     ):
 512         """
 513         Modifies an entry at database
 514         :param table: collection or table
 515         :param q_filter: Filter
 516         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 517         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 518         it raises a DbException
 519         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 520                       ignored. If not exist, it is ignored
 521         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 522                      if exist in the array is removed. If not exist, it is ignored
 523         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 524         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 525                      is appended to the end of the array
 526         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 527                           whole array
 528         :return: Dict with the number of entries modified. None if no matching is found.
 529         """
 530         with self.lock:
 531             for i, db_item in self._find(table, self._format_filter(q_filter)):
 532                 updated = self._update(
 533                     db_item,
 534                     update_dict,
 535                     unset=unset,
 536                     pull=pull,
 537                     push=push,
 538                     push_list=push_list,
 539                     pull_list=pull_list,
 540                 )
 541                 return {"updated": 1 if updated else 0}
 542             else:
 543                 if fail_on_empty:
 544                     raise DbException(
 545                         "Not found entry with _id='{}'".format(q_filter),
 546                         HTTPStatus.NOT_FOUND,
 547                     )
 548                 return None
 549
 550     def set_list(
 551         self,
 552         table,
 553         q_filter,
 554         update_dict,
 555         unset=None,
 556         pull=None,
 557         push=None,
 558         push_list=None,
 559         pull_list=None,
 560     ):
 561         """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
 562         with self.lock:
 563             updated = 0
 564             found = 0
 565             for _, db_item in self._find(table, self._format_filter(q_filter)):
 566                 found += 1
 567                 if self._update(
 568                     db_item,
 569                     update_dict,
 570                     unset=unset,
 571                     pull=pull,
 572                     push=push,
 573                     push_list=push_list,
 574                     pull_list=pull_list,
 575                 ):
 576                     updated += 1
 577             # if not found and fail_on_empty:
 578             #     raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
 579             return {"updated": updated} if found else None
 580
 581     def replace(self, table, _id, indata, fail_on_empty=True):
 582         """
 583         Replace the content of an entry
 584         :param table: collection or table
 585         :param _id: internal database id
 586         :param indata: content to replace
 587         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 588         it raises a DbException
 589         :return: Dict with the number of entries replaced
 590         """
 591         try:
 592             with self.lock:
 593                 for i, _ in self._find(table, self._format_filter({"_id": _id})):
 594                     break
 595                 else:
 596                     if fail_on_empty:
 597                         raise DbException(
 598                             "Not found entry with _id='{}'".format(_id),
 599                             HTTPStatus.NOT_FOUND,
 600                         )
 601                     return None
 602                 self.db[table][i] = deepcopy(indata)
 603             return {"updated": 1}
 604         except DbException:
 605             raise
 606         except Exception as e:  # TODO refine
 607             raise DbException(str(e))
 608
 609     def create(self, table, indata):
 610         """
 611         Add a new entry at database
 612         :param table: collection or table
 613         :param indata: content to be added
 614         :return: database '_id' of the inserted element. Raises a DbException on error
 615         """
 616         try:
 617             id = indata.get("_id")
 618             if not id:
 619                 id = str(uuid4())
 620                 indata["_id"] = id
 621             with self.lock:
 622                 if table not in self.db:
 623                     self.db[table] = []
 624                 self.db[table].append(deepcopy(indata))
 625             return id
 626         except Exception as e:  # TODO refine
 627             raise DbException(str(e))
 628
 629     def create_list(self, table, indata_list):
 630         """
 631         Add a new entry at database
 632         :param table: collection or table
 633         :param indata_list: list content to be added
 634         :return: list of inserted 'id's. Raises a DbException on error
 635         """
 636         try:
 637             _ids = []
 638             with self.lock:
 639                 for indata in indata_list:
 640                     _id = indata.get("_id")
 641                     if not _id:
 642                         _id = str(uuid4())
 643                         indata["_id"] = _id
 644                     with self.lock:
 645                         if table not in self.db:
 646                             self.db[table] = []
 647                         self.db[table].append(deepcopy(indata))
 648                     _ids.append(_id)
 649             return _ids
 650         except Exception as e:  # TODO refine
 651             raise DbException(str(e))
 652
 653
 654 if __name__ == "__main__":
 655     # some test code
 656     db = DbMemory()
 657     db.create("test", {"_id": 1, "data": 1})
 658     db.create("test", {"_id": 2, "data": 2})
 659     db.create("test", {"_id": 3, "data": 3})
 660     print("must be 3 items:", db.get_list("test"))
 661     print("must return item 2:", db.get_list("test", {"_id": 2}))
 662     db.del_one("test", {"_id": 2})
 663     print("must be emtpy:", db.get_list("test", {"_id": 2}))