From: tierno Date: Mon, 8 Oct 2018 14:30:15 +0000 (+0200) Subject: Added set_list at database. X-Git-Tag: v5.0.0~15 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fcommon.git;a=commitdiff_plain;h=87858cab98b3b169fc891fd2e0a0ba10f8b46127 Added set_list at database. fix bug 543 Added lincense header. Initial encrypt/decrypt methods Change-Id: I46194961d927ff008f3774d8d157587b06114be5 Signed-off-by: tierno --- diff --git a/.gitignore-common b/.gitignore-common index 609762b..0019f5a 100644 --- a/.gitignore-common +++ b/.gitignore-common @@ -1,3 +1,18 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This is a template with common files to be igonored, after clone make a copy to .gitignore # cp .gitignore-common .gitignore diff --git a/Dockerfile b/Dockerfile index 4ec7ecd..ef59cb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM ubuntu:16.04 RUN apt-get update && apt-get -y install git make python python3 python3-stdeb \ diff --git a/Jenkinsfile b/Jenkinsfile index 517e6ea..816b66c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + properties([ parameters([ string(defaultValue: env.BRANCH_NAME, description: '', name: 'GERRIT_BRANCH'), diff --git a/Makefile b/Makefile index a568202..410db79 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,17 @@ +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. all: clean package diff --git a/README.rst b/README.rst index b947507..1a7d7a1 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,18 @@ +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + =========== osm-common =========== diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh index 727c020..1b98637 100755 --- a/devops-stages/stage-archive.sh +++ b/devops-stages/stage-archive.sh @@ -1,4 +1,18 @@ #!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + MDG=common rm -rf pool rm -rf dists diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 2a93b0a..2e05cd1 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -1,5 +1,18 @@ #!/bin/sh +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # moved to a Makefile in order to add post install. Needed for "pip3 install aiokafka", # that is not available with a package make clean package diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index c6623f8..b866c8e 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -1,3 +1,17 @@ #!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + tox -e flake8 diff --git a/osm_common/__init__.py b/osm_common/__init__.py index 882df08..d283da5 100644 --- a/osm_common/__init__.py +++ b/osm_common/__init__.py @@ -1,2 +1,19 @@ -version = '0.1.7' -date_version = '2018-09-07' +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = '0.1.8' +date_version = '2018-10-09' diff --git a/osm_common/dbbase.py b/osm_common/dbbase.py index 3700b3a..cdb1644 100644 --- a/osm_common/dbbase.py +++ b/osm_common/dbbase.py @@ -1,4 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import yaml +import logging from http import HTTPStatus from copy import deepcopy @@ -8,39 +26,145 @@ __author__ = "Alfonso Tierno " class DbException(Exception): def __init__(self, message, http_code=HTTPStatus.NOT_FOUND): - # TODO change to http.HTTPStatus instead of int that allows .value and .name self.http_code = http_code - Exception.__init__(self, "database exception " + message) + Exception.__init__(self, "database exception " + str(message)) class DbBase(object): - def __init__(self): - pass + def __init__(self, logger_name='db', master_password=None): + """ + Constructor od dbBase + :param logger_name: logging name + :param master_password: master password used for encrypt decrypt methods + """ + self.logger = logging.getLogger(logger_name) + self.master_password = master_password def db_connect(self, config): + """ + Connect to database + :param config: Configuration of database + :return: None or raises DbException on error + """ pass def db_disconnect(self): + """ + Disconnect from database + :return: None + """ pass - def get_list(self, table, filter={}): + def get_list(self, table, q_filter=None): + """ + Obtain a list of entries matching q_filter + :param table: collection or table + :param q_filter: Filter + :return: a list (can be empty) with the found entries. Raises DbException on error + """ raise DbException("Method 'get_list' not implemented") - def get_one(self, table, filter={}, fail_on_empty=True, fail_on_more=True): + def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True): + """ + Obtain one entry matching q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so + that it raises a DbException + :return: The requested element, or None + """ raise DbException("Method 'get_one' not implemented") + def del_list(self, table, q_filter=None): + """ + Deletes all entries that match q_filter + :param table: collection or table + :param q_filter: Filter + :return: Dict with the number of entries deleted + """ + raise DbException("Method 'del_list' not implemented") + + def del_one(self, table, q_filter=None, fail_on_empty=True): + """ + Deletes one entry that matches q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in + which case it raises a DbException + :return: Dict with the number of entries deleted + """ + raise DbException("Method 'del_one' not implemented") + def create(self, table, indata): + """ + Add a new entry at database + :param table: collection or table + :param indata: content to be added + :return: database id of the inserted element. Raises a DbException on error + """ raise DbException("Method 'create' not implemented") - def del_list(self, table, filter={}): - raise DbException("Method 'del_list' not implemented") + def set_one(self, table, q_filter, update_dict, fail_on_empty=True): + """ + Modifies an entry at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :return: Dict with the number of entries modified. None if no matching is found. + """ + raise DbException("Method 'set_one' not implemented") - def del_one(self, table, filter={}, fail_on_empty=True): - raise DbException("Method 'del_one' not implemented") + def set_list(self, table, q_filter, update_dict): + """ + Modifies al matching entries at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :return: Dict with the number of entries modified + """ + raise DbException("Method 'set_list' not implemented") + + def replace(self, table, _id, indata, fail_on_empty=True): + """ + Replace the content of an entry + :param table: collection or table + :param _id: internal database id + :param indata: content to replace + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :return: Dict with the number of entries replaced + """ + raise DbException("Method 'replace' not implemented") + + def encrypt(self, value, salt=None): + """ + Encrypt a value + :param value: value to be encrypted + :param salt: optional salt to be used + :return: Encrypted content of value + """ + # for the moment return same value. until all modules call this method + return value + # raise DbException("Method 'encrypt' not implemented") + def decrypt(self, value, salt=None): + """ + Decrypt an encrypted value + :param value: value to be decrypted + :param salt: optional salt to be used + :return: Plain content of value + """ + # for the moment return same value. until all modules call this method + return value + # raise DbException("Method 'decrypt' not implemented") -def deep_update(dict_to_change, dict_reference, key_list=None): + +def deep_update_rfc7396(dict_to_change, dict_reference, key_list=None): """ Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396 Basically is a recursive python 'dict_to_change.update(dict_reference)', but a value of None is used to delete. @@ -181,14 +305,14 @@ def deep_update(dict_to_change, dict_reference, key_list=None): elif not isinstance(values_to_edit_delete[index], dict): # NotDict->Anything array_to_change[index] = deepcopy(values_to_edit_delete[index]) elif isinstance(array_to_change[index], dict): # Dict->Dict - deep_update(array_to_change[index], values_to_edit_delete[index], _key_list) + deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list) else: # Dict->NotDict if isinstance(array_to_change[index], list): # Dict->List. Check extra array edition if _deep_update_array(array_to_change[index], values_to_edit_delete[index], _key_list): continue array_to_change[index] = deepcopy(values_to_edit_delete[index]) - # calling deep_update to delete the None values - deep_update(array_to_change[index], values_to_edit_delete[index], _key_list) + # calling deep_update_rfc7396 to delete the None values + deep_update_rfc7396(array_to_change[index], values_to_edit_delete[index], _key_list) except IndexError: raise DbException("Array edition index out of range at '{}'".format(":".join(_key_list))) @@ -203,8 +327,8 @@ def deep_update(dict_to_change, dict_reference, key_list=None): _key_list[-1] = str(k) insert_value_copy = deepcopy(insert_value) if isinstance(insert_value_copy, dict): - # calling deep_update to delete the None values - deep_update(insert_value_copy, insert_value, _key_list) + # calling deep_update_rfc7396 to delete the None values + deep_update_rfc7396(insert_value_copy, insert_value, _key_list) array_to_change.append(insert_value_copy) _key_list.pop() @@ -224,15 +348,20 @@ def deep_update(dict_to_change, dict_reference, key_list=None): dict_to_change[k] = deepcopy(dict_reference[k]) elif k not in dict_to_change: # Dict->Empty dict_to_change[k] = deepcopy(dict_reference[k]) - # calling deep_update to delete the None values - deep_update(dict_to_change[k], dict_reference[k], key_list) + # calling deep_update_rfc7396 to delete the None values + deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list) elif isinstance(dict_to_change[k], dict): # Dict->Dict - deep_update(dict_to_change[k], dict_reference[k], key_list) + deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list) else: # Dict->NotDict if isinstance(dict_to_change[k], list): # Dict->List. Check extra array edition if _deep_update_array(dict_to_change[k], dict_reference[k], key_list): continue dict_to_change[k] = deepcopy(dict_reference[k]) - # calling deep_update to delete the None values - deep_update(dict_to_change[k], dict_reference[k], key_list) + # calling deep_update_rfc7396 to delete the None values + deep_update_rfc7396(dict_to_change[k], dict_reference[k], key_list) key_list.pop() + + +def deep_update(dict_to_change, dict_reference): + """ Maintained for backward compatibility. Use deep_update_rfc7396 instead""" + return deep_update_rfc7396(dict_to_change, dict_reference) diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index 63c93c1..e37d97b 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -1,3 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from osm_common.dbbase import DbException, DbBase from http import HTTPStatus @@ -9,32 +26,43 @@ __author__ = "Alfonso Tierno " class DbMemory(DbBase): - def __init__(self, logger_name='db'): - self.logger = logging.getLogger(logger_name) + def __init__(self, logger_name='db', master_password=None): + super().__init__(logger_name, master_password) self.db = {} def db_connect(self, config): + """ + Connect to database + :param config: Configuration of database + :return: None or raises DbException on error + """ if "logger_name" in config: self.logger = logging.getLogger(config["logger_name"]) @staticmethod - def _format_filter(filter): - return filter # TODO + def _format_filter(q_filter): + return q_filter # TODO - def _find(self, table, filter): + def _find(self, table, q_filter): for i, row in enumerate(self.db.get(table, ())): match = True - if filter: - for k, v in filter.items(): + if q_filter: + for k, v in q_filter.items(): if k not in row or v != row[k]: match = False if match: yield i, row - def get_list(self, table, filter={}): + def get_list(self, table, q_filter=None): + """ + Obtain a list of entries matching q_filter + :param table: collection or table + :param q_filter: Filter + :return: a list (can be empty) with the found entries. Raises DbException on error + """ try: result = [] - for _, row in self._find(table, self._format_filter(filter)): + for _, row in self._find(table, self._format_filter(q_filter)): result.append(deepcopy(row)) return result except DbException: @@ -42,26 +70,42 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def get_one(self, table, filter={}, fail_on_empty=True, fail_on_more=True): + def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True): + """ + Obtain one entry matching q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so + that it raises a DbException + :return: The requested element, or None + """ try: result = None - for _, row in self._find(table, self._format_filter(filter)): + for _, row in self._find(table, self._format_filter(q_filter)): if not fail_on_more: return deepcopy(row) if result: - raise DbException("Found more than one entry with filter='{}'".format(filter), + raise DbException("Found more than one entry with filter='{}'".format(q_filter), HTTPStatus.CONFLICT.value) result = row if not result and fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) return deepcopy(result) except Exception as e: # TODO refine raise DbException(str(e)) - def del_list(self, table, filter={}): + def del_list(self, table, q_filter=None): + """ + Deletes all entries that match q_filter + :param table: collection or table + :param q_filter: Filter + :return: Dict with the number of entries deleted + """ try: id_list = [] - for i, _ in self._find(table, self._format_filter(filter)): + for i, _ in self._find(table, self._format_filter(q_filter)): id_list.append(i) deleted = len(id_list) for i in reversed(id_list): @@ -72,26 +116,43 @@ class DbMemory(DbBase): except Exception as e: # TODO refine raise DbException(str(e)) - def del_one(self, table, filter={}, fail_on_empty=True): + def del_one(self, table, q_filter=None, fail_on_empty=True): + """ + Deletes one entry that matches q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in + which case it raises a DbException + :return: Dict with the number of entries deleted + """ try: - for i, _ in self._find(table, self._format_filter(filter)): + for i, _ in self._find(table, self._format_filter(q_filter)): break else: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found entry with filter='{}'".format(q_filter), HTTPStatus.NOT_FOUND) return None del self.db[table][i] return {"deleted": 1} except Exception as e: # TODO refine raise DbException(str(e)) - def replace(self, table, filter, indata, fail_on_empty=True): + def replace(self, table, _id, indata, fail_on_empty=True): + """ + Replace the content of an entry + :param table: collection or table + :param _id: internal database id + :param indata: content to replace + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :return: Dict with the number of entries replaced + """ try: - for i, _ in self._find(table, self._format_filter(filter)): + for i, _ in self._find(table, self._format_filter({"_id": _id})): break else: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found entry with _id='{}'".format(_id), HTTPStatus.NOT_FOUND) return None self.db[table][i] = deepcopy(indata) return {"updated": 1} @@ -99,6 +160,12 @@ class DbMemory(DbBase): raise DbException(str(e)) def create(self, table, indata): + """ + Add a new entry at database + :param table: collection or table + :param indata: content to be added + :return: database id of the inserted element. Raises a DbException on error + """ try: id = indata.get("_id") if not id: diff --git a/osm_common/dbmongo.py b/osm_common/dbmongo.py index 1d56e58..9b5bc57 100644 --- a/osm_common/dbmongo.py +++ b/osm_common/dbmongo.py @@ -1,3 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from pymongo import MongoClient, errors @@ -18,14 +35,15 @@ __author__ = "Alfonso Tierno " # return call(*args, **kwargs) # except pymongo.AutoReconnect as e: # if retry == 4: -# raise DbException(str(e)) +# raise DbException(e) # sleep(retry) # return _retry_mongocall def deep_update(to_update, update_with): """ - Update 'to_update' dict with the content 'update_with' dict recursively + Similar to deepcopy but recursively with nested dictionaries. 'to_update' dict is updated with a content copy of + 'update_with' dict recursively :param to_update: must be a dictionary to be modified :param update_with: must be a dictionary. It is not changed :return: to_update @@ -43,10 +61,17 @@ class DbMongo(DbBase): conn_initial_timout = 120 conn_timout = 10 - def __init__(self, logger_name='db'): - self.logger = logging.getLogger(logger_name) + def __init__(self, logger_name='db', master_password=None): + super().__init__(logger_name, master_password) + self.client = None + self.db = None def db_connect(self, config): + """ + Connect to database + :param config: Configuration of database + :return: None or raises DbException on error + """ try: if "logger_name" in config: self.logger = logging.getLogger(config["logger_name"]) @@ -66,15 +91,12 @@ class DbMongo(DbBase): self.logger.info("Waiting to database up {}".format(e)) sleep(2) except errors.PyMongoError as e: - raise DbException(str(e)) - - def db_disconnect(self): - pass # TODO + raise DbException(e) @staticmethod def _format_filter(q_filter): """ - Translate query string filter into mongo database filter + Translate query string q_filter into mongo database filter :param q_filter: Query string content. Follows SOL005 section 4.3.2 guidelines, with the follow extensions and differences: It accept ".nq" (not equal) in addition to ".neq". @@ -106,6 +128,8 @@ class DbMongo(DbBase): """ try: db_filter = {} + if not q_filter: + return db_filter for query_k, query_v in q_filter.items(): dot_index = query_k.rfind(".") if dot_index > 1 and query_k[dot_index+1:] in ("eq", "ne", "gt", "gte", "lt", "lte", "cont", @@ -154,11 +178,17 @@ class DbMongo(DbBase): raise DbException("Invalid query string filter at {}:{}. Error: {}".format(query_k, v, e), http_code=HTTPStatus.BAD_REQUEST) - def get_list(self, table, filter={}): + def get_list(self, table, q_filter=None): + """ + Obtain a list of entries matching q_filter + :param table: collection or table + :param q_filter: Filter + :return: a list (can be empty) with the found entries. Raises DbException on error + """ try: result = [] collection = self.db[table] - db_filter = self._format_filter(filter) + db_filter = self._format_filter(q_filter) rows = collection.find(db_filter) for row in rows: result.append(row) @@ -166,83 +196,144 @@ class DbMongo(DbBase): except DbException: raise except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) - def get_one(self, table, filter={}, fail_on_empty=True, fail_on_more=True): + def get_one(self, table, q_filter=None, fail_on_empty=True, fail_on_more=True): + """ + Obtain one entry matching q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :param fail_on_more: If more than one matches filter it returns one of then unless this flag is set tu True, so + that it raises a DbException + :return: The requested element, or None + """ try: - if filter: - filter = self._format_filter(filter) + db_filter = self._format_filter(q_filter) collection = self.db[table] if not (fail_on_empty and fail_on_more): - return collection.find_one(filter) - rows = collection.find(filter) + return collection.find_one(db_filter) + rows = collection.find(db_filter) if rows.count() == 0: if fail_on_empty: - raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + raise DbException("Not found any {} with filter='{}'".format(table[:-1], q_filter), HTTPStatus.NOT_FOUND) return None elif rows.count() > 1: if fail_on_more: - raise DbException("Found more than one {} with filter='{}'".format(table[:-1], filter), + raise DbException("Found more than one {} with filter='{}'".format(table[:-1], q_filter), HTTPStatus.CONFLICT) return rows[0] except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) - def del_list(self, table, filter={}): + def del_list(self, table, q_filter=None): + """ + Deletes all entries that match q_filter + :param table: collection or table + :param q_filter: Filter + :return: Dict with the number of entries deleted + """ try: collection = self.db[table] - rows = collection.delete_many(self._format_filter(filter)) + rows = collection.delete_many(self._format_filter(q_filter)) return {"deleted": rows.deleted_count} except DbException: raise except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) - def del_one(self, table, filter={}, fail_on_empty=True): + def del_one(self, table, q_filter=None, fail_on_empty=True): + """ + Deletes one entry that matches q_filter + :param table: collection or table + :param q_filter: Filter + :param fail_on_empty: If nothing matches filter it returns '0' deleted unless this flag is set tu True, in + which case it raises a DbException + :return: Dict with the number of entries deleted + """ try: collection = self.db[table] - rows = collection.delete_one(self._format_filter(filter)) + rows = collection.delete_one(self._format_filter(q_filter)) if rows.deleted_count == 0: if fail_on_empty: - raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + raise DbException("Not found any {} with filter='{}'".format(table[:-1], q_filter), HTTPStatus.NOT_FOUND) return None return {"deleted": rows.deleted_count} except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) def create(self, table, indata): + """ + Add a new entry at database + :param table: collection or table + :param indata: content to be added + :return: database id of the inserted element. Raises a DbException on error + """ try: collection = self.db[table] data = collection.insert_one(indata) return data.inserted_id except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) - def set_one(self, table, filter, update_dict, fail_on_empty=True): + def set_one(self, table, q_filter, update_dict, fail_on_empty=True): + """ + Modifies an entry at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :return: Dict with the number of entries modified. None if no matching is found. + """ try: collection = self.db[table] - rows = collection.update_one(self._format_filter(filter), {"$set": update_dict}) + rows = collection.update_one(self._format_filter(q_filter), {"$set": update_dict}) if rows.matched_count == 0: if fail_on_empty: - raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + raise DbException("Not found any {} with filter='{}'".format(table[:-1], q_filter), HTTPStatus.NOT_FOUND) return None return {"modified": rows.modified_count} except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) + + def set_list(self, table, q_filter, update_dict): + """ + Modifies al matching entries at database + :param table: collection or table + :param q_filter: Filter + :param update_dict: Plain dictionary with the content to be updated. It is a dot separated keys and a value + :return: Dict with the number of entries modified + """ + try: + collection = self.db[table] + rows = collection.update_many(self._format_filter(q_filter), {"$set": update_dict}) + return {"modified": rows.modified_count} + except Exception as e: # TODO refine + raise DbException(e) - def replace(self, table, id, indata, fail_on_empty=True): + def replace(self, table, _id, indata, fail_on_empty=True): + """ + Replace the content of an entry + :param table: collection or table + :param _id: internal database id + :param indata: content to replace + :param fail_on_empty: If nothing matches filter it returns None unless this flag is set tu True, in which case + it raises a DbException + :return: Dict with the number of entries replaced + """ try: - _filter = {"_id": id} + db_filter = {"_id": _id} collection = self.db[table] - rows = collection.replace_one(_filter, indata) + rows = collection.replace_one(db_filter, indata) if rows.matched_count == 0: if fail_on_empty: - raise DbException("Not found any {} with filter='{}'".format(table[:-1], _filter), - HTTPStatus.NOT_FOUND) + raise DbException("Not found any {} with _id='{}'".format(table[:-1], _id), HTTPStatus.NOT_FOUND) return None return {"replaced": rows.modified_count} except Exception as e: # TODO refine - raise DbException(str(e)) + raise DbException(e) diff --git a/osm_common/fsbase.py b/osm_common/fsbase.py index e975409..6f82cd3 100644 --- a/osm_common/fsbase.py +++ b/osm_common/fsbase.py @@ -1,3 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from http import HTTPStatus diff --git a/osm_common/fslocal.py b/osm_common/fslocal.py index 202872d..a027558 100644 --- a/osm_common/fslocal.py +++ b/osm_common/fslocal.py @@ -1,3 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import logging # import tarfile diff --git a/osm_common/msgbase.py b/osm_common/msgbase.py index 2b338df..8978085 100644 --- a/osm_common/msgbase.py +++ b/osm_common/msgbase.py @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. # import asyncio from http import HTTPStatus diff --git a/osm_common/msgkafka.py b/osm_common/msgkafka.py index 3b38956..aa756c4 100644 --- a/osm_common/msgkafka.py +++ b/osm_common/msgkafka.py @@ -1,3 +1,18 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import asyncio import yaml diff --git a/osm_common/msglocal.py b/osm_common/msglocal.py index 75fc717..f731e74 100644 --- a/osm_common/msglocal.py +++ b/osm_common/msglocal.py @@ -1,3 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import os import yaml diff --git a/setup.py b/setup.py index c10a4f0..62390b6 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,21 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from setuptools import setup diff --git a/tox.ini b/tox.ini index 12f309b..e3bc42c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,18 @@ +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [tox] envlist = py27,py3,flake8,pytest toxworkdir={homedir}/.tox