osm_common/dbmemory.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Telefonica S.A.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 from copy import deepcopy
  19 from http import HTTPStatus
  20 import logging
  21 from uuid import uuid4
  22
  23 from osm_common.dbbase import DbBase, DbException
  24 from osm_common.dbmongo import deep_update
  25
  26
  27 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  28
  29
  30 class DbMemory(DbBase):
  31     def __init__(self, logger_name="db", lock=False):
  32         super().__init__(logger_name, lock)
  33         self.db = {}
  34
  35     def db_connect(self, config):
  36         """
  37         Connect to database
  38         :param config: Configuration of database
  39         :return: None or raises DbException on error
  40         """
  41         if "logger_name" in config:
  42             self.logger = logging.getLogger(config["logger_name"])
  43         master_key = config.get("commonkey") or config.get("masterpassword")
  44         if master_key:
  45             self.set_secret_key(master_key)
  46
  47     @staticmethod
  48     def _format_filter(q_filter):
  49         db_filter = {}
  50         # split keys with ANYINDEX in this way:
  51         # {"A.B.ANYINDEX.C.D.ANYINDEX.E": v }  -> {"A.B.ANYINDEX": {"C.D.ANYINDEX": {"E": v}}}
  52         if q_filter:
  53             for k, v in q_filter.items():
  54                 db_v = v
  55                 kleft, _, kright = k.rpartition(".ANYINDEX.")
  56                 while kleft:
  57                     k = kleft + ".ANYINDEX"
  58                     db_v = {kright: db_v}
  59                     kleft, _, kright = k.rpartition(".ANYINDEX.")
  60                 deep_update(db_filter, {k: db_v})
  61
  62         return db_filter
  63
  64     def _find(self, table, q_filter):
  65         def recursive_find(key_list, key_next_index, content, oper, target):
  66             if key_next_index == len(key_list) or content is None:
  67                 try:
  68                     if oper in ("eq", "cont"):
  69                         if isinstance(target, list):
  70                             if isinstance(content, list):
  71                                 return any(
  72                                     content_item in target for content_item in content
  73                                 )
  74                             return content in target
  75                         elif isinstance(content, list):
  76                             return target in content
  77                         else:
  78                             return content == target
  79                     elif oper in ("neq", "ne", "ncont"):
  80                         if isinstance(target, list):
  81                             if isinstance(content, list):
  82                                 return all(
  83                                     content_item not in target
  84                                     for content_item in content
  85                                 )
  86                             return content not in target
  87                         elif isinstance(content, list):
  88                             return target not in content
  89                         else:
  90                             return content != target
  91                     if oper == "gt":
  92                         return content > target
  93                     elif oper == "gte":
  94                         return content >= target
  95                     elif oper == "lt":
  96                         return content < target
  97                     elif oper == "lte":
  98                         return content <= target
  99                     else:
 100                         raise DbException(
 101                             "Unknown filter operator '{}' in key '{}'".format(
 102                                 oper, ".".join(key_list)
 103                             ),
 104                             http_code=HTTPStatus.BAD_REQUEST,
 105                         )
 106                 except TypeError:
 107                     return False
 108
 109             elif isinstance(content, dict):
 110                 return recursive_find(
 111                     key_list,
 112                     key_next_index + 1,
 113                     content.get(key_list[key_next_index]),
 114                     oper,
 115                     target,
 116                 )
 117             elif isinstance(content, list):
 118                 look_for_match = True  # when there is a match return immediately
 119                 if (target is None) != (
 120                     oper in ("neq", "ne", "ncont")
 121                 ):  # one True and other False (Xor)
 122                     look_for_match = (
 123                         False  # when there is not a match return immediately
 124                     )
 125
 126                 for content_item in content:
 127                     if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict):
 128                         matches = True
 129                         for k2, v2 in target.items():
 130                             k_new_list = k2.split(".")
 131                             new_operator = "eq"
 132                             if k_new_list[-1] in (
 133                                 "eq",
 134                                 "ne",
 135                                 "gt",
 136                                 "gte",
 137                                 "lt",
 138                                 "lte",
 139                                 "cont",
 140                                 "ncont",
 141                                 "neq",
 142                             ):
 143                                 new_operator = k_new_list.pop()
 144                             if not recursive_find(
 145                                 k_new_list, 0, content_item, new_operator, v2
 146                             ):
 147                                 matches = False
 148                                 break
 149
 150                     else:
 151                         matches = recursive_find(
 152                             key_list, key_next_index, content_item, oper, target
 153                         )
 154                     if matches == look_for_match:
 155                         return matches
 156                 if key_list[key_next_index].isdecimal() and int(
 157                     key_list[key_next_index]
 158                 ) < len(content):
 159                     matches = recursive_find(
 160                         key_list,
 161                         key_next_index + 1,
 162                         content[int(key_list[key_next_index])],
 163                         oper,
 164                         target,
 165                     )
 166                     if matches == look_for_match:
 167                         return matches
 168                 return not look_for_match
 169             else:  # content is not dict, nor list neither None, so not found
 170                 if oper in ("neq", "ne", "ncont"):
 171                     return target is not None
 172                 else:
 173                     return target is None
 174
 175         for i, row in enumerate(self.db.get(table, ())):
 176             q_filter = q_filter or {}
 177             for k, v in q_filter.items():
 178                 k_list = k.split(".")
 179                 operator = "eq"
 180                 if k_list[-1] in (
 181                     "eq",
 182                     "ne",
 183                     "gt",
 184                     "gte",
 185                     "lt",
 186                     "lte",
 187                     "cont",
 188                     "ncont",
 189                     "neq",
 190                 ):
 191                     operator = k_list.pop()
 192                 matches = recursive_find(k_list, 0, row, operator, v)
 193                 if not matches:
 194                     break
 195             else:
 196                 # match
 197                 yield i, row
 198
 199     def get_list(self, table, q_filter=None):
 200         """
 201         Obtain a list of entries matching q_filter
 202         :param table: collection or table
 203         :param q_filter: Filter
 204         :return: a list (can be empty) with the found entries. Raises DbException on error
 205         """
 206         try:
 207             result = []
 208             with self.lock:
 209                 for _, row in self._find(table, self._format_filter(q_filter)):
 210                     result.append(deepcopy(row))
 211             return result
 212         except DbException:
 213             raise
 214         except Exception as e:  # TODO refine
 215             raise DbException(str(e))
 216
 217     def count(self, table, q_filter=None):
 218         """
 219         Count the number of entries matching q_filter
 220         :param table: collection or table
 221         :param q_filter: Filter
 222         :return: number of entries found (can be zero)
 223         :raise: DbException on error
 224         """
 225         try:
 226             with self.lock:
 227                 return sum(1 for x in self._find(table, self._format_filter(q_filter)))
 228         except DbException:
 229             raise
 230         except Exception as e:  # TODO refine
 231             raise DbException(str(e))
 232
 233     def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True):
 234         """
 235         Obtain one entry matching q_filter
 236         :param table: collection or table
 237         :param q_filter: Filter
 238         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 239         it raises a DbException
 240         :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so
 241         that it raises a DbException
 242         :return: The requested element, or None
 243         """
 244         try:
 245             result = None
 246             with self.lock:
 247                 for _, row in self._find(table, self._format_filter(q_filter)):
 248                     if not fail_on_more:
 249                         return deepcopy(row)
 250                     if result:
 251                         raise DbException(
 252                             "Found more than one entry with filter='{}'".format(
 253                                 q_filter
 254                             ),
 255                             HTTPStatus.CONFLICT.value,
 256                         )
 257                     result = row
 258             if not result and fail_on_empty:
 259                 raise DbException(
 260                     "Not found entry with filter='{}'".format(q_filter),
 261                     HTTPStatus.NOT_FOUND,
 262                 )
 263             return deepcopy(result)
 264         except Exception as e:  # TODO refine
 265             raise DbException(str(e))
 266
 267     def del_list(self, table, q_filter=None):
 268         """
 269         Deletes all entries that match q_filter
 270         :param table: collection or table
 271         :param q_filter: Filter
 272         :return: Dict with the number of entries deleted
 273         """
 274         try:
 275             id_list = []
 276             with self.lock:
 277                 for i, _ in self._find(table, self._format_filter(q_filter)):
 278                     id_list.append(i)
 279             deleted = len(id_list)
 280             for i in reversed(id_list):
 281                 del self.db[table][i]
 282             return {"deleted": deleted}
 283         except DbException:
 284             raise
 285         except Exception as e:  # TODO refine
 286             raise DbException(str(e))
 287
 288     def del_one(self, table, q_filter=None, fail_on_empty=True):
 289         """
 290         Deletes one entry that matches q_filter
 291         :param table: collection or table
 292         :param q_filter: Filter
 293         :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in
 294         which case it raises a DbException
 295         :return: Dict with the number of entries deleted
 296         """
 297         try:
 298             with self.lock:
 299                 for i, _ in self._find(table, self._format_filter(q_filter)):
 300                     break
 301                 else:
 302                     if fail_on_empty:
 303                         raise DbException(
 304                             "Not found entry with filter='{}'".format(q_filter),
 305                             HTTPStatus.NOT_FOUND,
 306                         )
 307                     return None
 308                 del self.db[table][i]
 309             return {"deleted": 1}
 310         except Exception as e:  # TODO refine
 311             raise DbException(str(e))
 312
 313     def _update(
 314         self,
 315         db_item,
 316         update_dict,
 317         unset=None,
 318         pull=None,
 319         push=None,
 320         push_list=None,
 321         pull_list=None,
 322     ):
 323         """
 324         Modifies an entry at database
 325         :param db_item: entry of the table to update
 326         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 327         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 328                       ignored. If not exist, it is ignored
 329         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 330                      if exist in the array is removed. If not exist, it is ignored
 331         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 332         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 333                      is appended to the end of the array
 334         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 335                           whole array
 336         :return: True if database has been changed, False if not; Exception on error
 337         """
 338
 339         def _iterate_keys(k, db_nested, populate=True):
 340             k_list = k.split(".")
 341             k_item_prev = k_list[0]
 342             populated = False
 343             if k_item_prev not in db_nested and populate:
 344                 populated = True
 345                 db_nested[k_item_prev] = None
 346             for k_item in k_list[1:]:
 347                 if isinstance(db_nested[k_item_prev], dict):
 348                     if k_item not in db_nested[k_item_prev]:
 349                         if not populate:
 350                             raise DbException(
 351                                 "Cannot set '{}', not existing '{}'".format(k, k_item)
 352                             )
 353                         populated = True
 354                         db_nested[k_item_prev][k_item] = None
 355                 elif isinstance(db_nested[k_item_prev], list) and k_item.isdigit():
 356                     # extend list with Nones if index greater than list
 357                     k_item = int(k_item)
 358                     if k_item >= len(db_nested[k_item_prev]):
 359                         if not populate:
 360                             raise DbException(
 361                                 "Cannot set '{}', index too large '{}'".format(
 362                                     k, k_item
 363                                 )
 364                             )
 365                         populated = True
 366                         db_nested[k_item_prev] += [None] * (
 367                             k_item - len(db_nested[k_item_prev]) + 1
 368                         )
 369                 elif db_nested[k_item_prev] is None:
 370                     if not populate:
 371                         raise DbException(
 372                             "Cannot set '{}', not existing '{}'".format(k, k_item)
 373                         )
 374                     populated = True
 375                     db_nested[k_item_prev] = {k_item: None}
 376                 else:  # number, string, boolean, ... or list but with not integer key
 377                     raise DbException(
 378                         "Cannot set '{}' on existing '{}={}'".format(
 379                             k, k_item_prev, db_nested[k_item_prev]
 380                         )
 381                     )
 382                 db_nested = db_nested[k_item_prev]
 383                 k_item_prev = k_item
 384             return db_nested, k_item_prev, populated
 385
 386         updated = False
 387         try:
 388             if update_dict:
 389                 for dot_k, v in update_dict.items():
 390                     dict_to_update, key_to_update, _ = _iterate_keys(dot_k, db_item)
 391                     dict_to_update[key_to_update] = v
 392                     updated = True
 393             if unset:
 394                 for dot_k in unset:
 395                     try:
 396                         dict_to_update, key_to_update, _ = _iterate_keys(
 397                             dot_k, db_item, populate=False
 398                         )
 399                         del dict_to_update[key_to_update]
 400                         updated = True
 401                     except Exception:
 402                         pass
 403             if pull:
 404                 for dot_k, v in pull.items():
 405                     try:
 406                         dict_to_update, key_to_update, _ = _iterate_keys(
 407                             dot_k, db_item, populate=False
 408                         )
 409                     except Exception:
 410                         continue
 411                     if key_to_update not in dict_to_update:
 412                         continue
 413                     if not isinstance(dict_to_update[key_to_update], list):
 414                         raise DbException(
 415                             "Cannot pull '{}'. Target is not a list".format(dot_k)
 416                         )
 417                     while v in dict_to_update[key_to_update]:
 418                         dict_to_update[key_to_update].remove(v)
 419                         updated = True
 420             if pull_list:
 421                 for dot_k, v in pull_list.items():
 422                     if not isinstance(v, list):
 423                         raise DbException(
 424                             "Invalid content at pull_list, '{}' must be an array".format(
 425                                 dot_k
 426                             ),
 427                             http_code=HTTPStatus.BAD_REQUEST,
 428                         )
 429                     try:
 430                         dict_to_update, key_to_update, _ = _iterate_keys(
 431                             dot_k, db_item, populate=False
 432                         )
 433                     except Exception:
 434                         continue
 435                     if key_to_update not in dict_to_update:
 436                         continue
 437                     if not isinstance(dict_to_update[key_to_update], list):
 438                         raise DbException(
 439                             "Cannot pull_list '{}'. Target is not a list".format(dot_k)
 440                         )
 441                     for single_v in v:
 442                         while single_v in dict_to_update[key_to_update]:
 443                             dict_to_update[key_to_update].remove(single_v)
 444                             updated = True
 445             if push:
 446                 for dot_k, v in push.items():
 447                     dict_to_update, key_to_update, populated = _iterate_keys(
 448                         dot_k, db_item
 449                     )
 450                     if (
 451                         isinstance(dict_to_update, dict)
 452                         and key_to_update not in dict_to_update
 453                     ):
 454                         dict_to_update[key_to_update] = [v]
 455                         updated = True
 456                     elif populated and dict_to_update[key_to_update] is None:
 457                         dict_to_update[key_to_update] = [v]
 458                         updated = True
 459                     elif not isinstance(dict_to_update[key_to_update], list):
 460                         raise DbException(
 461                             "Cannot push '{}'. Target is not a list".format(dot_k)
 462                         )
 463                     else:
 464                         dict_to_update[key_to_update].append(v)
 465                         updated = True
 466             if push_list:
 467                 for dot_k, v in push_list.items():
 468                     if not isinstance(v, list):
 469                         raise DbException(
 470                             "Invalid content at push_list, '{}' must be an array".format(
 471                                 dot_k
 472                             ),
 473                             http_code=HTTPStatus.BAD_REQUEST,
 474                         )
 475                     dict_to_update, key_to_update, populated = _iterate_keys(
 476                         dot_k, db_item
 477                     )
 478                     if (
 479                         isinstance(dict_to_update, dict)
 480                         and key_to_update not in dict_to_update
 481                     ):
 482                         dict_to_update[key_to_update] = v.copy()
 483                         updated = True
 484                     elif populated and dict_to_update[key_to_update] is None:
 485                         dict_to_update[key_to_update] = v.copy()
 486                         updated = True
 487                     elif not isinstance(dict_to_update[key_to_update], list):
 488                         raise DbException(
 489                             "Cannot push '{}'. Target is not a list".format(dot_k),
 490                             http_code=HTTPStatus.CONFLICT,
 491                         )
 492                     else:
 493                         dict_to_update[key_to_update] += v
 494                         updated = True
 495
 496             return updated
 497         except DbException:
 498             raise
 499         except Exception as e:  # TODO refine
 500             raise DbException(str(e))
 501
 502     def set_one(
 503         self,
 504         table,
 505         q_filter,
 506         update_dict,
 507         fail_on_empty=True,
 508         unset=None,
 509         pull=None,
 510         push=None,
 511         push_list=None,
 512         pull_list=None,
 513     ):
 514         """
 515         Modifies an entry at database
 516         :param table: collection or table
 517         :param q_filter: Filter
 518         :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value
 519         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 520         it raises a DbException
 521         :param unset: Plain dictionary with the content to be removed if exist. It is a dot separated keys, value is
 522                       ignored. If not exist, it is ignored
 523         :param pull: Plain dictionary with the content to be removed from an array. It is a dot separated keys and value
 524                      if exist in the array is removed. If not exist, it is ignored
 525         :param pull_list: Same as pull but values are arrays where each item is removed from the array
 526         :param push: Plain dictionary with the content to be appended to an array. It is a dot separated keys and value
 527                      is appended to the end of the array
 528         :param push_list: Same as push but values are arrays where each item is and appended instead of appending the
 529                           whole array
 530         :return: Dict with the number of entries modified. None if no matching is found.
 531         """
 532         with self.lock:
 533             for i, db_item in self._find(table, self._format_filter(q_filter)):
 534                 updated = self._update(
 535                     db_item,
 536                     update_dict,
 537                     unset=unset,
 538                     pull=pull,
 539                     push=push,
 540                     push_list=push_list,
 541                     pull_list=pull_list,
 542                 )
 543                 return {"updated": 1 if updated else 0}
 544             else:
 545                 if fail_on_empty:
 546                     raise DbException(
 547                         "Not found entry with _id='{}'".format(q_filter),
 548                         HTTPStatus.NOT_FOUND,
 549                     )
 550                 return None
 551
 552     def set_list(
 553         self,
 554         table,
 555         q_filter,
 556         update_dict,
 557         unset=None,
 558         pull=None,
 559         push=None,
 560         push_list=None,
 561         pull_list=None,
 562     ):
 563         """Modifies al matching entries at database. Same as push. Do not fail if nothing matches"""
 564         with self.lock:
 565             updated = 0
 566             found = 0
 567             for _, db_item in self._find(table, self._format_filter(q_filter)):
 568                 found += 1
 569                 if self._update(
 570                     db_item,
 571                     update_dict,
 572                     unset=unset,
 573                     pull=pull,
 574                     push=push,
 575                     push_list=push_list,
 576                     pull_list=pull_list,
 577                 ):
 578                     updated += 1
 579             # if not found and fail_on_empty:
 580             #     raise DbException("Not found entry with '{}'".format(q_filter), HTTPStatus.NOT_FOUND)
 581             return {"updated": updated} if found else None
 582
 583     def replace(self, table, _id, indata, fail_on_empty=True):
 584         """
 585         Replace the content of an entry
 586         :param table: collection or table
 587         :param _id: internal database id
 588         :param indata: content to replace
 589         :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case
 590         it raises a DbException
 591         :return: Dict with the number of entries replaced
 592         """
 593         try:
 594             with self.lock:
 595                 for i, _ in self._find(table, self._format_filter({"_id": _id})):
 596                     break
 597                 else:
 598                     if fail_on_empty:
 599                         raise DbException(
 600                             "Not found entry with _id='{}'".format(_id),
 601                             HTTPStatus.NOT_FOUND,
 602                         )
 603                     return None
 604                 self.db[table][i] = deepcopy(indata)
 605             return {"updated": 1}
 606         except DbException:
 607             raise
 608         except Exception as e:  # TODO refine
 609             raise DbException(str(e))
 610
 611     def create(self, table, indata):
 612         """
 613         Add a new entry at database
 614         :param table: collection or table
 615         :param indata: content to be added
 616         :return: database '_id' of the inserted element. Raises a DbException on error
 617         """
 618         try:
 619             id = indata.get("_id")
 620             if not id:
 621                 id = str(uuid4())
 622                 indata["_id"] = id
 623             with self.lock:
 624                 if table not in self.db:
 625                     self.db[table] = []
 626                 self.db[table].append(deepcopy(indata))
 627             return id
 628         except Exception as e:  # TODO refine
 629             raise DbException(str(e))
 630
 631     def create_list(self, table, indata_list):
 632         """
 633         Add a new entry at database
 634         :param table: collection or table
 635         :param indata_list: list content to be added
 636         :return: list of inserted 'id's. Raises a DbException on error
 637         """
 638         try:
 639             _ids = []
 640             with self.lock:
 641                 for indata in indata_list:
 642                     _id = indata.get("_id")
 643                     if not _id:
 644                         _id = str(uuid4())
 645                         indata["_id"] = _id
 646                     with self.lock:
 647                         if table not in self.db:
 648                             self.db[table] = []
 649                         self.db[table].append(deepcopy(indata))
 650                     _ids.append(_id)
 651             return _ids
 652         except Exception as e:  # TODO refine
 653             raise DbException(str(e))
 654
 655
 656 if __name__ == "__main__":
 657     # some test code
 658     db = DbMemory()
 659     db.create("test", {"_id": 1, "data": 1})
 660     db.create("test", {"_id": 2, "data": 2})
 661     db.create("test", {"_id": 3, "data": 3})
 662     print("must be 3 items:", db.get_list("test"))
 663     print("must return item 2:", db.get_list("test", {"_id": 2}))
 664     db.del_one("test", {"_id": 2})
 665     print("must be emtpy:", db.get_list("test", {"_id": 2}))