From: Dario Faccin Date: Wed, 24 May 2023 10:48:37 +0000 (+0200) Subject: Update from master X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=37e0914881759a514424ad5c0cc9278b9ced27a2;p=osm%2Fcommon.git Update from master Squashed commit of the following: commit b5015160aca7f04f2b0fb35c87281c0ab480f429 Author: Pedro Escaleira Date: Wed May 17 00:13:05 2023 +0100 Bug 2246 fixed Change-Id: Ic35d131d4e44686de207b0a6acc3360e04306cb2 Signed-off-by: Pedro Escaleira commit 01df3ee231471330760e03b013382464e773eee2 Author: Gulsum Atici Date: Thu May 11 11:07:54 2023 +0300 Fix AttributeError caused by updated pymongo version AttributeError: 'Cursor' object has no attribute 'count' is fixed. Change-Id: Id1b9133376d5a7dcb3998c623163bb57dd5c534b Signed-off-by: Gulsum Atici commit 8f3ab9a82608ffe74e6fd5d0c532822412dbc88a Author: k4.rahul Date: Fri May 5 14:18:47 2023 +0530 Coverity-CWE 22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal') Coverity fix for 137960 Filesystem path, filename, or URI manipulation Change-Id: I0691a9f231d6b7019fe413c261f50262ea7fb923 Signed-off-by: k4.rahul commit bc94e3403ebab364fff7603c1a353c793b52966e Author: k4.rahul Date: Fri May 5 15:52:47 2023 +0530 Coverity-CWE 260: Password in Configuration File Hardcoded credentials in​ configuration file​ remove from the cloud-config.txt file as it is not being used anywhere in the test case Change-Id: I101e4b9b6f48fa6d34822bc6f400552329f9aa18 Signed-off-by: k4.rahul commit db28d4290a34ed1a7ac7a2ca10cab7eb34a55fd1 Author: garciadeblas Date: Wed May 10 16:26:55 2023 +0200 Minor updates in Dockerfile Change-Id: Ia12406fef38b13c56ebec3be5bee53cd00441181 Signed-off-by: garciadeblas commit 0edc5108ac7e584cac210ed6bae8b8ef09511388 Author: garciadeblas Date: Tue Apr 18 15:07:15 2023 +0200 Clean stage-archive.sh and use allowlist_extenals in tox.ini Change-Id: I511cd5009563589f54899f667d779239745f2778 Signed-off-by: garciadeblas commit a06b854f2b278aaee015fc1f76015895f8cf50c1 Author: Gulsum Atici Date: Tue May 9 13:42:13 2023 +0300 Ubuntu 22.04 and Python 3.10 preparation Change-Id: I740202d48977467a0c2b2afb4b17bd7597331dee Signed-off-by: Gulsum Atici commit b2d732a70efa33e4bc478d351d64bc4adb4ea332 Author: k4.rahul Date: Thu Apr 27 16:20:47 2023 +0530 Coverity-CWE 476: NULL Pointer Dereference (137978 Bad use of null-like value) Coverity fix for Bad use of null-like value Change-Id: I7437c2b2aeeff25619b5405bcd7c962f3fbd70bb Signed-off-by: k4.rahul commit 09496abf441b0f3730f0288df161da1ca004be69 Author: Gabriel Cuba Date: Tue Apr 4 01:57:17 2023 -0500 Fix bug 2231: reverse_sync in fsmongo obtains file timestamp with UTC timezone instead of local timezone, so that filesystem and mongodb timestamps are correctly compared Change-Id: I0056026704a624329aae1ae52a45143d12f6dfdd Signed-off-by: Gabriel Cuba commit 76394efe9fbee088dddd1dc9d4da6f043c3959a5 Author: Gulsum Atici Date: Mon Jan 9 23:19:18 2023 +0300 Feature 10950 Replace pycrypto with pycryptodome Remove the pycrypto library and change encrypt and decrypt methods to work with pycryptodome. Move encryption methods from N2VC to common. Change-Id: I12a5f6138664ab6ebb7100c82523e91750f05f14 Signed-off-by: Gulsum Atici Change-Id: Iccf3fcafe14305099fcceef65efd6e3c57cf6d2f Signed-off-by: Dario Faccin --- diff --git a/Dockerfile b/Dockerfile index f99b758..27ab273 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ # devops-stages/stage-build.sh # -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -37,10 +37,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ python3 \ python3-all \ python3-dev \ - python3-setuptools - -RUN python3 -m easy_install pip==21.3.1 -RUN pip install tox==3.24.5 + python3-setuptools \ + python3-pip \ + tox ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh index 1b98637..34a613c 100755 --- a/devops-stages/stage-archive.sh +++ b/devops-stages/stage-archive.sh @@ -18,7 +18,4 @@ rm -rf pool rm -rf dists mkdir -p pool/$MDG mv deb_dist/*.deb pool/$MDG/ -mkdir -p dists/unstable/$MDG/binary-amd64/ -apt-ftparchive packages pool/$MDG > dists/unstable/$MDG/binary-amd64/Packages -gzip -9fk dists/unstable/$MDG/binary-amd64/Packages -echo "dists/**,pool/$MDG/*.deb" + diff --git a/osm_common/dbbase.py b/osm_common/dbbase.py index 6b3a89a..d0d4fb0 100644 --- a/osm_common/dbbase.py +++ b/osm_common/dbbase.py @@ -14,7 +14,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -import asyncio + from base64 import b64decode, b64encode from copy import deepcopy from http import HTTPStatus @@ -673,18 +673,16 @@ def deep_update(dict_to_change, dict_reference): class Encryption(DbBase): - def __init__(self, uri, config, encoding_type="ascii", loop=None, logger_name="db"): + def __init__(self, uri, config, encoding_type="ascii", logger_name="db"): """Constructor. Args: uri (str): Connection string to connect to the database. config (dict): Additional database info encoding_type (str): ascii, utf-8 etc. - loop (object): Asyncio Loop logger_name (str): Logger name """ - self.loop = loop or asyncio.get_event_loop() self._secret_key = None # 32 bytes length array used for encrypt/decrypt self.encrypt_mode = AES.MODE_ECB super(Encryption, self).__init__( diff --git a/osm_common/dbmemory.py b/osm_common/dbmemory.py index 272f6d6..e72db5b 100644 --- a/osm_common/dbmemory.py +++ b/osm_common/dbmemory.py @@ -126,26 +126,27 @@ class DbMemory(DbBase): for content_item in content: if key_list[key_next_index] == "ANYINDEX" and isinstance(v, dict): matches = True - for k2, v2 in target.items(): - k_new_list = k2.split(".") - new_operator = "eq" - if k_new_list[-1] in ( - "eq", - "ne", - "gt", - "gte", - "lt", - "lte", - "cont", - "ncont", - "neq", - ): - new_operator = k_new_list.pop() - if not recursive_find( - k_new_list, 0, content_item, new_operator, v2 - ): - matches = False - break + if target: + for k2, v2 in target.items(): + k_new_list = k2.split(".") + new_operator = "eq" + if k_new_list[-1] in ( + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "cont", + "ncont", + "neq", + ): + new_operator = k_new_list.pop() + if not recursive_find( + k_new_list, 0, content_item, new_operator, v2 + ): + matches = False + break else: matches = recursive_find( diff --git a/osm_common/dbmongo.py b/osm_common/dbmongo.py index f5c4d30..e5e12c6 100644 --- a/osm_common/dbmongo.py +++ b/osm_common/dbmongo.py @@ -284,7 +284,7 @@ class DbMongo(DbBase): with self.lock: collection = self.db[table] db_filter = self._format_filter(q_filter) - count = collection.count(db_filter) + count = collection.count_documents(db_filter) return count except DbException: raise @@ -308,8 +308,8 @@ class DbMongo(DbBase): collection = self.db[table] if not (fail_on_empty and fail_on_more): return collection.find_one(db_filter) - rows = collection.find(db_filter) - if rows.count() == 0: + rows = list(collection.find(db_filter)) + if len(rows) == 0: if fail_on_empty: raise DbException( "Not found any {} with filter='{}'".format( @@ -317,8 +317,9 @@ class DbMongo(DbBase): ), HTTPStatus.NOT_FOUND, ) + return None - elif rows.count() > 1: + elif len(rows) > 1: if fail_on_more: raise DbException( "Found more than one {} with filter='{}'".format( diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py index f99267f..2e47039 100644 --- a/osm_common/fsmongo.py +++ b/osm_common/fsmongo.py @@ -235,7 +235,9 @@ class FsMongo(FsBase): if e.errno != errno.ENOENT: # This is probably permission denied or worse raise - os.symlink(link, file_path) + os.symlink( + link, os.path.realpath(os.path.normpath(os.path.abspath(file_path))) + ) else: folder = os.path.dirname(file_path) if folder not in valid_paths: @@ -601,7 +603,9 @@ class FsMongo(FsBase): # convert to relative path rel_filename = os.path.relpath(member["filename"], self.path) - last_modified_date = datetime.datetime.fromtimestamp( + # get timestamp in UTC because mongo stores upload date in UTC: + # https://www.mongodb.com/docs/v4.0/tutorial/model-time-data/#overview + last_modified_date = datetime.datetime.utcfromtimestamp( os.path.getmtime(member["filename"]) ) diff --git a/osm_common/msgbase.py b/osm_common/msgbase.py index 80c5be5..958bb3e 100644 --- a/osm_common/msgbase.py +++ b/osm_common/msgbase.py @@ -79,14 +79,14 @@ class MsgBase(object): "Method 'read' not implemented", http_code=HTTPStatus.INTERNAL_SERVER_ERROR ) - async def aiowrite(self, topic, key, msg, loop=None): + async def aiowrite(self, topic, key, msg): raise MsgException( "Method 'aiowrite' not implemented", http_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) async def aioread( - self, topic, loop=None, callback=None, aiocallback=None, group_id=None, **kwargs + self, topic, callback=None, aiocallback=None, group_id=None, **kwargs ): raise MsgException( "Method 'aioread' not implemented", diff --git a/osm_common/msgkafka.py b/osm_common/msgkafka.py index 5487093..02b8241 100644 --- a/osm_common/msgkafka.py +++ b/osm_common/msgkafka.py @@ -35,7 +35,6 @@ class MsgKafka(MsgBase): self.port = None self.consumer = None self.producer = None - self.loop = None self.broker = None self.group_id = None @@ -45,7 +44,6 @@ class MsgKafka(MsgBase): self.logger = logging.getLogger(config["logger_name"]) self.host = config["host"] self.port = config["port"] - self.loop = config.get("loop") or asyncio.get_event_loop() self.broker = str(self.host) + ":" + str(self.port) self.group_id = config.get("group_id") @@ -55,7 +53,6 @@ class MsgKafka(MsgBase): def disconnect(self): try: pass - # self.loop.close() except Exception as e: # TODO refine raise MsgException(str(e)) @@ -70,9 +67,7 @@ class MsgKafka(MsgBase): retry = 2 # Try two times while retry: try: - self.loop.run_until_complete( - self.aiowrite(topic=topic, key=key, msg=msg) - ) + asyncio.run(self.aiowrite(topic=topic, key=key, msg=msg)) break except Exception as e: retry -= 1 @@ -88,27 +83,22 @@ class MsgKafka(MsgBase): :return: topic, key, message; or None """ try: - return self.loop.run_until_complete(self.aioread(topic, self.loop)) + return asyncio.run(self.aioread(topic)) except MsgException: raise except Exception as e: raise MsgException("Error reading {} topic: {}".format(topic, str(e))) - async def aiowrite(self, topic, key, msg, loop=None): + async def aiowrite(self, topic, key, msg): """ Asyncio write :param topic: str kafka topic :param key: str kafka key :param msg: str or dictionary kafka message - :param loop: asyncio loop. To be DEPRECATED! in near future!!! loop must be provided inside config at connect :return: None """ - - if not loop: - loop = self.loop try: self.producer = AIOKafkaProducer( - loop=loop, key_serializer=str.encode, value_serializer=str.encode, bootstrap_servers=self.broker, @@ -127,7 +117,6 @@ class MsgKafka(MsgBase): async def aioread( self, topic, - loop=None, callback=None, aiocallback=None, group_id=None, @@ -137,7 +126,6 @@ class MsgKafka(MsgBase): """ Asyncio read from one or several topics. :param topic: can be str: single topic; or str list: several topics - :param loop: asyncio loop. To be DEPRECATED! in near future!!! loop must be provided inside config at connect :param callback: synchronous callback function that will handle the message in kafka bus :param aiocallback: async callback function that will handle the message in kafka bus :param group_id: kafka group_id to use. Can be False (set group_id to None), None (use general group_id provided @@ -148,9 +136,6 @@ class MsgKafka(MsgBase): :param kwargs: optional keyword arguments for callback function :return: If no callback defined, it returns (topic, key, message) """ - - if not loop: - loop = self.loop if group_id is False: group_id = None elif group_id is None: @@ -161,7 +146,6 @@ class MsgKafka(MsgBase): else: topic_list = (topic,) self.consumer = AIOKafkaConsumer( - loop=loop, bootstrap_servers=self.broker, group_id=group_id, auto_offset_reset="earliest" if from_beginning else "latest", diff --git a/osm_common/msglocal.py b/osm_common/msglocal.py index 6d4cb58..0c3b216 100644 --- a/osm_common/msglocal.py +++ b/osm_common/msglocal.py @@ -41,7 +41,6 @@ class MsgLocal(MsgBase): self.files_read = {} self.files_write = {} self.buffer = {} - self.loop = None def connect(self, config): try: @@ -52,7 +51,6 @@ class MsgLocal(MsgBase): self.path += "/" if not os.path.exists(self.path): os.mkdir(self.path) - self.loop = config.get("loop") except MsgException: raise @@ -158,12 +156,11 @@ class MsgLocal(MsgBase): raise MsgException(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) async def aioread( - self, topic, loop=None, callback=None, aiocallback=None, group_id=None, **kwargs + self, topic, callback=None, aiocallback=None, group_id=None, **kwargs ): """ Asyncio read from one or several topics. It blocks :param topic: can be str: single topic; or str list: several topics - :param loop: asyncio loop. To be DEPRECATED! in near future!!! loop must be provided inside config at connect :param callback: synchronous callback function that will handle the message :param aiocallback: async callback function that will handle the message :param group_id: group_id to use for load balancing. Can be False (set group_id to None), None (use general @@ -171,7 +168,6 @@ class MsgLocal(MsgBase): :param kwargs: optional keyword arguments for callback function :return: If no callback defined, it returns (topic, key, message) """ - _loop = loop or self.loop try: while True: msg = self.read(topic, blocks=False) @@ -182,19 +178,18 @@ class MsgLocal(MsgBase): await aiocallback(*msg, **kwargs) else: return msg - await asyncio.sleep(2, loop=_loop) + await asyncio.sleep(2) except MsgException: raise except Exception as e: # TODO refine raise MsgException(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - async def aiowrite(self, topic, key, msg, loop=None): + async def aiowrite(self, topic, key, msg): """ Asyncio write. It blocks :param topic: str :param key: str :param msg: message, can be str or yaml - :param loop: asyncio loop :return: nothing if ok or raises an exception """ return self.write(topic, key, msg) diff --git a/osm_common/tests/packages/invalid_package_vnf/Scripts/cloud_init/cloud-config.txt b/osm_common/tests/packages/invalid_package_vnf/Scripts/cloud_init/cloud-config.txt index 7a83e12..5c78ae3 100755 --- a/osm_common/tests/packages/invalid_package_vnf/Scripts/cloud_init/cloud-config.txt +++ b/osm_common/tests/packages/invalid_package_vnf/Scripts/cloud_init/cloud-config.txt @@ -19,7 +19,6 @@ # #cloud-config -password: osm4u chpasswd: { expire: False } ssh_pwauth: True diff --git a/osm_common/tests/packages/native_charm_with_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt b/osm_common/tests/packages/native_charm_with_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt index f5d56f6..1e33dd4 100755 --- a/osm_common/tests/packages/native_charm_with_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt +++ b/osm_common/tests/packages/native_charm_with_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt @@ -20,7 +20,6 @@ #cloud-config -password: osm4u chpasswd: { expire: False } ssh_pwauth: True diff --git a/osm_common/tests/packages/native_charm_without_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt b/osm_common/tests/packages/native_charm_without_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt index f5d56f6..1e33dd4 100755 --- a/osm_common/tests/packages/native_charm_without_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt +++ b/osm_common/tests/packages/native_charm_without_metadata_dir_vnf/Scripts/cloud_init/cloud-config.txt @@ -20,7 +20,6 @@ #cloud-config -password: osm4u chpasswd: { expire: False } ssh_pwauth: True diff --git a/osm_common/tests/test_dbbase.py b/osm_common/tests/test_dbbase.py index 050abdb..e582c7e 100644 --- a/osm_common/tests/test_dbbase.py +++ b/osm_common/tests/test_dbbase.py @@ -997,7 +997,6 @@ class TestAsyncEncryption(unittest.TestCase): def setUp(self, mock_logger): mock_logger = logging.getLogger() mock_logger.disabled = True - self.loop = asyncio.get_event_loop() self.encryption = Encryption(uri="uri", config={}) self.encryption.encoding_type = encoding_type self.encryption.encrypt_mode = encyrpt_mode @@ -1022,7 +1021,8 @@ class TestAsyncEncryption(unittest.TestCase): "ip": "192.168.12.23", } fields = ["secret", "cacert"] - self.loop.run_until_complete( + + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual(input_item, expected_item) @@ -1035,7 +1035,7 @@ class TestAsyncEncryption(unittest.TestCase): """item is empty and fields exists.""" input_item = {} fields = ["secret", "cacert"] - self.loop.run_until_complete( + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual(input_item, {}) @@ -1046,7 +1046,7 @@ class TestAsyncEncryption(unittest.TestCase): """item exists and fields is empty.""" input_item = copy.deepcopy(item) fields = [] - self.loop.run_until_complete( + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual(input_item, item) @@ -1064,7 +1064,7 @@ class TestAsyncEncryption(unittest.TestCase): "path": "/var", "ip": "192.168.12.23", } - self.loop.run_until_complete( + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual(input_item, expected_item) @@ -1081,7 +1081,7 @@ class TestAsyncEncryption(unittest.TestCase): mock_decrypt.return_value = "mysecret" input_item = copy.deepcopy(item) fields = ["secret"] - self.loop.run_until_complete( + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual(input_item, item) @@ -1097,7 +1097,7 @@ class TestAsyncEncryption(unittest.TestCase): fields = ["secret"] input_item = copy.deepcopy(item) with self.assertRaises(Exception) as error: - self.loop.run_until_complete( + asyncio.run( self.encryption.decrypt_fields(input_item, fields, schema_version, salt) ) self.assertEqual( @@ -1113,9 +1113,7 @@ class TestAsyncEncryption(unittest.TestCase): def test_encrypt(self, mock_encrypt_value, mock_get_secret_key): """Method decrypt raises error.""" mock_encrypt_value.return_value = encyrpted_value - result = self.loop.run_until_complete( - self.encryption.encrypt(value, schema_version, salt) - ) + result = asyncio.run(self.encryption.encrypt(value, schema_version, salt)) self.assertEqual(result, encyrpted_value) mock_get_secret_key.assert_called_once() mock_encrypt_value.assert_called_once_with(value, schema_version, salt) @@ -1128,9 +1126,7 @@ class TestAsyncEncryption(unittest.TestCase): """Method get_secret_key raises error.""" mock_get_secret_key.side_effect = DbException("Unexpected type.") with self.assertRaises(Exception) as error: - self.loop.run_until_complete( - self.encryption.encrypt(value, schema_version, salt) - ) + asyncio.run(self.encryption.encrypt(value, schema_version, salt)) self.assertEqual(str(error.exception), "database exception Unexpected type.") mock_get_secret_key.assert_called_once() mock_encrypt_value.assert_not_called() @@ -1143,9 +1139,7 @@ class TestAsyncEncryption(unittest.TestCase): "A bytes-like object is required, not 'str'" ) with self.assertRaises(Exception) as error: - self.loop.run_until_complete( - self.encryption.encrypt(value, schema_version, salt) - ) + asyncio.run(self.encryption.encrypt(value, schema_version, salt)) self.assertEqual( str(error.exception), "A bytes-like object is required, not 'str'" ) @@ -1157,7 +1151,7 @@ class TestAsyncEncryption(unittest.TestCase): def test_decrypt(self, mock_decrypt_value, mock_get_secret_key): """Decrypted successfully.""" mock_decrypt_value.return_value = value - result = self.loop.run_until_complete( + result = asyncio.run( self.encryption.decrypt(encyrpted_value, schema_version, salt) ) self.assertEqual(result, value) @@ -1174,9 +1168,7 @@ class TestAsyncEncryption(unittest.TestCase): """Method get_secret_key raises error.""" mock_get_secret_key.side_effect = DbException("Unexpected type.") with self.assertRaises(Exception) as error: - self.loop.run_until_complete( - self.encryption.decrypt(encyrpted_value, schema_version, salt) - ) + asyncio.run(self.encryption.decrypt(encyrpted_value, schema_version, salt)) self.assertEqual(str(error.exception), "database exception Unexpected type.") mock_get_secret_key.assert_called_once() mock_decrypt_value.assert_not_called() @@ -1191,9 +1183,7 @@ class TestAsyncEncryption(unittest.TestCase): "A bytes-like object is required, not 'str'" ) with self.assertRaises(Exception) as error: - self.loop.run_until_complete( - self.encryption.decrypt(encyrpted_value, schema_version, salt) - ) + asyncio.run(self.encryption.decrypt(encyrpted_value, schema_version, salt)) self.assertEqual( str(error.exception), "A bytes-like object is required, not 'str'" ) @@ -1281,7 +1271,7 @@ class TestAsyncEncryption(unittest.TestCase): def test_get_secret_key_exists(self, mock_join_keys): """secret_key exists.""" self.encryption._secret_key = secret_key - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(self.encryption.secret_key, secret_key) mock_join_keys.assert_not_called() @@ -1295,7 +1285,7 @@ class TestAsyncEncryption(unittest.TestCase): self.encryption._admin_collection.find_one.return_value = None self.encryption._config = {"database_commonkey": "osm_new_key"} mock_join_keys.return_value = joined_key - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(self.encryption.secret_key, joined_key) self.assertEqual(mock_join_keys.call_count, 1) mock_b64decode.assert_not_called() @@ -1310,7 +1300,7 @@ class TestAsyncEncryption(unittest.TestCase): self.encryption._admin_collection.find_one.return_value = {"version": "1.0"} self.encryption._config = {"database_commonkey": "osm_new_key"} mock_join_keys.return_value = joined_key - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(self.encryption.secret_key, joined_key) self.assertEqual(mock_join_keys.call_count, 1) mock_b64decode.assert_not_called() @@ -1335,7 +1325,7 @@ class TestAsyncEncryption(unittest.TestCase): self.encryption._config = {"database_commonkey": "osm_new_key"} mock_join_keys.side_effect = [secret_key, joined_key] mock_b64decode.return_value = base64_decoded_serial - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(self.encryption.secret_key, joined_key) self.assertEqual(mock_join_keys.call_count, 2) mock_b64decode.assert_called_once_with(serial_bytes) @@ -1360,7 +1350,7 @@ class TestAsyncEncryption(unittest.TestCase): self.encryption._config = {"database_commonkey": "osm_new_key"} mock_join_keys.side_effect = DbException("Invalid data type.") with self.assertRaises(Exception) as error: - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(str(error.exception), "database exception Invalid data type.") self.assertEqual(mock_join_keys.call_count, 1) check_if_assert_not_called( @@ -1384,7 +1374,7 @@ class TestAsyncEncryption(unittest.TestCase): "A bytes-like object is required, not 'str'" ) with self.assertRaises(Exception) as error: - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual( str(error.exception), "A bytes-like object is required, not 'str'" ) @@ -1410,7 +1400,7 @@ class TestAsyncEncryption(unittest.TestCase): self.encryption._config = {"database_commonkey": "osm_new_key"} mock_join_keys.return_value = secret_key with self.assertRaises(Exception) as error: - self.loop.run_until_complete(self.encryption.get_secret_key()) + asyncio.run(self.encryption.get_secret_key()) self.assertEqual(str(error.exception), "database exception Connection failed.") self.assertEqual(self.encryption.secret_key, None) self.assertEqual(mock_join_keys.call_count, 1) @@ -1423,10 +1413,10 @@ class TestAsyncEncryption(unittest.TestCase): def test_encrypt_decrypt_with_schema_version_1_1_with_salt(self): """Encrypt and decrypt with schema version 1.1, salt exists.""" - encrypted_msg = self.loop.run_until_complete( + encrypted_msg = asyncio.run( self.encryption.encrypt(value, schema_version, salt) ) - decrypted_msg = self.loop.run_until_complete( + decrypted_msg = asyncio.run( self.encryption.decrypt(encrypted_msg, schema_version, salt) ) self.assertEqual(value, decrypted_msg) @@ -1434,10 +1424,10 @@ class TestAsyncEncryption(unittest.TestCase): def test_encrypt_decrypt_with_schema_version_1_0_with_salt(self): """Encrypt and decrypt with schema version 1.0, salt exists.""" schema_version = "1.0" - encrypted_msg = self.loop.run_until_complete( + encrypted_msg = asyncio.run( self.encryption.encrypt(value, schema_version, salt) ) - decrypted_msg = self.loop.run_until_complete( + decrypted_msg = asyncio.run( self.encryption.decrypt(encrypted_msg, schema_version, salt) ) self.assertEqual(value, decrypted_msg) @@ -1446,9 +1436,7 @@ class TestAsyncEncryption(unittest.TestCase): """Encrypt and decrypt with schema version 1.1, without salt.""" salt = None with self.assertRaises(Exception) as error: - self.loop.run_until_complete( - self.encryption.encrypt(value, schema_version, salt) - ) + asyncio.run(self.encryption.encrypt(value, schema_version, salt)) self.assertEqual(str(error.exception), "'NoneType' object is not iterable") diff --git a/osm_common/tests/test_msgbase.py b/osm_common/tests/test_msgbase.py index d5092b1..41def48 100644 --- a/osm_common/tests/test_msgbase.py +++ b/osm_common/tests/test_msgbase.py @@ -16,7 +16,7 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: esousa@whitestack.com or alfonso.tiernosepulveda@telefonica.com ## - +import asyncio import http from osm_common.msgbase import MsgBase, MsgException @@ -64,20 +64,18 @@ def test_read(msg_base): assert excinfo.value.http_code == http.HTTPStatus.INTERNAL_SERVER_ERROR -def test_aiowrite(msg_base, event_loop): +def test_aiowrite(msg_base): with pytest.raises(MsgException) as excinfo: - event_loop.run_until_complete( - msg_base.aiowrite("test", "test", "test", event_loop) - ) + asyncio.run(msg_base.aiowrite("test", "test", "test")) assert str(excinfo.value).startswith( exception_message("Method 'aiowrite' not implemented") ) assert excinfo.value.http_code == http.HTTPStatus.INTERNAL_SERVER_ERROR -def test_aioread(msg_base, event_loop): +def test_aioread(msg_base): with pytest.raises(MsgException) as excinfo: - event_loop.run_until_complete(msg_base.aioread("test", event_loop)) + asyncio.run(msg_base.aioread("test")) assert str(excinfo.value).startswith( exception_message("Method 'aioread' not implemented") ) diff --git a/osm_common/tests/test_msglocal.py b/osm_common/tests/test_msglocal.py index fb74586..b40b75c 100644 --- a/osm_common/tests/test_msglocal.py +++ b/osm_common/tests/test_msglocal.py @@ -16,7 +16,7 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: esousa@whitestack.com or alfonso.tiernosepulveda@telefonica.com ## - +import asyncio import http import logging import os @@ -437,7 +437,7 @@ def test_read_exception(msg_local_with_data, blocks): (["topic", "topic1", "topic2"], [{"key": "value"}, {"key1": "value1"}]), ], ) -def test_aioread(msg_local_with_data, event_loop, topics, datas): +def test_aioread(msg_local_with_data, topics, datas): def write_to_topic(topics, datas): time.sleep(2) for topic in topics: @@ -455,9 +455,7 @@ def test_aioread(msg_local_with_data, event_loop, topics, datas): t.start() for topic in topics: for data in datas: - recv = event_loop.run_until_complete( - msg_local_with_data.aioread(topic, event_loop) - ) + recv = asyncio.run(msg_local_with_data.aioread(topic)) recv_topic, recv_key, recv_msg = recv key = list(data.keys())[0] val = data[key] @@ -467,22 +465,22 @@ def test_aioread(msg_local_with_data, event_loop, topics, datas): t.join() -def test_aioread_exception(msg_local_with_data, event_loop): +def test_aioread_exception(msg_local_with_data): msg_local_with_data.files_read = MagicMock() msg_local_with_data.files_read.__contains__.side_effect = Exception() with pytest.raises(MsgException) as excinfo: - event_loop.run_until_complete(msg_local_with_data.aioread("topic1", event_loop)) + asyncio.run(msg_local_with_data.aioread("topic1")) assert str(excinfo.value).startswith(empty_exception_message()) assert excinfo.value.http_code == http.HTTPStatus.INTERNAL_SERVER_ERROR -def test_aioread_general_exception(msg_local_with_data, event_loop): +def test_aioread_general_exception(msg_local_with_data): msg_local_with_data.read = MagicMock() msg_local_with_data.read.side_effect = Exception() with pytest.raises(MsgException) as excinfo: - event_loop.run_until_complete(msg_local_with_data.aioread("topic1", event_loop)) + asyncio.run(msg_local_with_data.aioread("topic1")) assert str(excinfo.value).startswith(empty_exception_message()) assert excinfo.value.http_code == http.HTTPStatus.INTERNAL_SERVER_ERROR @@ -503,9 +501,9 @@ def test_aioread_general_exception(msg_local_with_data, event_loop): ("test_topic", "test_none", None), ], ) -def test_aiowrite(msg_local_config, event_loop, topic, key, msg): +def test_aiowrite(msg_local_config, topic, key, msg): file_path = msg_local_config.path + topic - event_loop.run_until_complete(msg_local_config.aiowrite(topic, key, msg)) + asyncio.run(msg_local_config.aiowrite(topic, key, msg)) assert os.path.exists(file_path) with open(file_path, "r") as stream: @@ -530,12 +528,10 @@ def test_aiowrite(msg_local_config, event_loop, topic, key, msg): ("test_topic", "test_none", None, 3), ], ) -def test_aiowrite_with_multiple_calls( - msg_local_config, event_loop, topic, key, msg, times -): +def test_aiowrite_with_multiple_calls(msg_local_config, topic, key, msg, times): file_path = msg_local_config.path + topic for _ in range(times): - event_loop.run_until_complete(msg_local_config.aiowrite(topic, key, msg)) + asyncio.run(msg_local_config.aiowrite(topic, key, msg)) assert os.path.exists(file_path) with open(file_path, "r") as stream: @@ -546,11 +542,11 @@ def test_aiowrite_with_multiple_calls( } -def test_aiowrite_exception(msg_local_config, event_loop): +def test_aiowrite_exception(msg_local_config): msg_local_config.files_write = MagicMock() msg_local_config.files_write.__contains__.side_effect = Exception() with pytest.raises(MsgException) as excinfo: - event_loop.run_until_complete(msg_local_config.aiowrite("test", "test", "test")) + asyncio.run(msg_local_config.aiowrite("test", "test", "test")) assert str(excinfo.value).startswith(empty_exception_message()) assert excinfo.value.http_code == http.HTTPStatus.INTERNAL_SERVER_ERROR diff --git a/releasenotes/notes/Fix_bug_2231-3b2a0cf186a460ce.yaml b/releasenotes/notes/Fix_bug_2231-3b2a0cf186a460ce.yaml new file mode 100644 index 0000000..6f27b8f --- /dev/null +++ b/releasenotes/notes/Fix_bug_2231-3b2a0cf186a460ce.yaml @@ -0,0 +1,21 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +fixes: + - | + Fixing bug 2231 - reverse_sync in fsmongo obtains file timestamp with UTC timezone instead of local timezone, so + that filesystem and mongodb timestamps are correctly compared \ No newline at end of file diff --git a/releasenotes/notes/cwe_476-c132043815560a45.yaml b/releasenotes/notes/cwe_476-c132043815560a45.yaml new file mode 100644 index 0000000..3a1966b --- /dev/null +++ b/releasenotes/notes/cwe_476-c132043815560a45.yaml @@ -0,0 +1,22 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +security: + - | + Coverity-CWE 476: NULL Pointer Dereference (137978 Bad use of null-like value) + Coverity fix for Bad use of null-like value + diff --git a/releasenotes/notes/fix_bug_2246-458a6a2380c018b9.yaml b/releasenotes/notes/fix_bug_2246-458a6a2380c018b9.yaml new file mode 100644 index 0000000..ce715f2 --- /dev/null +++ b/releasenotes/notes/fix_bug_2246-458a6a2380c018b9.yaml @@ -0,0 +1,20 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +fixes: + - | + Fixes bug 2246: Edited `osm_common/dbmongo.py` file, by substituting the `count` call by a `count_documents` in the `count` method. This is necessary to work with the PyMongo version 4.3.3 diff --git a/releasenotes/notes/fix_count_attribute_error-067438fb5fc5ef76.yaml b/releasenotes/notes/fix_count_attribute_error-067438fb5fc5ef76.yaml new file mode 100644 index 0000000..8a7ac98 --- /dev/null +++ b/releasenotes/notes/fix_count_attribute_error-067438fb5fc5ef76.yaml @@ -0,0 +1,21 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +fixes: + - | + Pymongo version upgrade caused and attribute error which is 'Cursor' object has no attribute 'count'. + The patch fixes this problem. diff --git a/releasenotes/notes/fix_cwe22-c2677929fb74c79d.yaml b/releasenotes/notes/fix_cwe22-c2677929fb74c79d.yaml new file mode 100644 index 0000000..9eb9a90 --- /dev/null +++ b/releasenotes/notes/fix_cwe22-c2677929fb74c79d.yaml @@ -0,0 +1,21 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +security: + - | + Coverity-CWE 22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal') + Coverity fix for 137960 Filesystem path, filename, or URI manipulation diff --git a/releasenotes/notes/fix_cwe_260-badcbfc255dd5e34.yaml b/releasenotes/notes/fix_cwe_260-badcbfc255dd5e34.yaml new file mode 100644 index 0000000..51f03a9 --- /dev/null +++ b/releasenotes/notes/fix_cwe_260-badcbfc255dd5e34.yaml @@ -0,0 +1,22 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +security: + - | + Coverity-CWE 260: Password in Configuration File + Hardcoded credentials in configuration file are removed from the cloud-config.txt file + as it is not being used anywhere in the test case. diff --git a/releasenotes/notes/rel14_preparation-ab2d5ede6a5bf28b.yaml b/releasenotes/notes/rel14_preparation-ab2d5ede6a5bf28b.yaml new file mode 100644 index 0000000..d1e756f --- /dev/null +++ b/releasenotes/notes/rel14_preparation-ab2d5ede6a5bf28b.yaml @@ -0,0 +1,21 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +upgrade: + - | + Upgrade Ubuntu from 20.04 to 22.04 and Python from 3.8 to 3.10. + diff --git a/requirements-test.txt b/requirements-test.txt index d31c36c..3201a4c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,19 +16,19 @@ ####################################################################################### attrs==22.2.0 # via pytest -coverage==7.2.1 +coverage==7.2.5 # via -r requirements-test.in -exceptiongroup==1.1.0 +exceptiongroup==1.1.1 # via pytest iniconfig==2.0.0 # via pytest -nose2==0.12.0 +nose2==0.13.0 # via -r requirements-test.in -packaging==23.0 +packaging==23.1 # via pytest pluggy==1.0.0 # via pytest -pytest==7.2.1 +pytest==7.3.1 # via # -r requirements-test.in # pytest-asyncio diff --git a/requirements.in b/requirements.in index 6031584..aa36ac5 100644 --- a/requirements.in +++ b/requirements.in @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -pymongo<4 +pymongo aiokafka charset-normalizer<4 # Required by aiohttp in LCM pyyaml==5.4.1 pycryptodome dataclasses -motor==1.3.1 +motor temporalio protobuf<4 # Required by Juju diff --git a/requirements.txt b/requirements.txt index c0f467d..9c1c424 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,15 +18,17 @@ aiokafka==0.8.0 # via -r requirements.in async-timeout==4.0.2 # via aiokafka -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 # via -r requirements.in dataclasses==0.6 # via -r requirements.in +dnspython==2.3.0 + # via pymongo kafka-python==2.0.2 # via aiokafka -motor==1.3.1 +motor==3.1.2 # via -r requirements.in -packaging==23.0 +packaging==23.1 # via aiokafka protobuf==3.20.3 # via @@ -34,7 +36,7 @@ protobuf==3.20.3 # temporalio pycryptodome==3.17 # via -r requirements.in -pymongo==3.13.0 +pymongo==4.3.3 # via # -r requirements.in # motor diff --git a/tox.ini b/tox.ini index 3a5b32b..4abe945 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ toxworkdir = /tmp/.tox [testenv] usedevelop = True -basepython = python3.8 +basepython = python3.10 setenv = VIRTUAL_ENV={envdir} PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -48,7 +48,7 @@ commands = coverage report --omit='*tests*' coverage html -d ./cover --omit='*tests*' coverage xml -o coverage.xml --omit=*tests* -whitelist_externals = sh +allowlist_externals = sh ####################################################################################### [testenv:flake8] @@ -80,7 +80,7 @@ commands = [testenv:pip-compile] deps = pip-tools==6.6.2 skip_install = true -whitelist_externals = bash +allowlist_externals = bash [ commands = - bash -c "for file in requirements*.in ; do \ @@ -103,13 +103,13 @@ commands = python3 setup.py --command-packages=stdeb.command sdist_dsc sh -c 'cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us' sh -c 'rm osm_common/requirements.txt' -whitelist_externals = sh +allowlist_externals = sh ####################################################################################### [testenv:release_notes] deps = reno skip_install = true -whitelist_externals = bash +allowlist_externals = bash commands = reno new {posargs:new_feature} bash -c "sed -i -e '1 e head -16 tox.ini' releasenotes/notes/{posargs:new_feature}*.yaml"