From 375aeb2647d733ac894b2408f66d36d55217c92d Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Wed, 10 May 2023 13:55:55 -0400 Subject: [PATCH] Update to Python 3.10 and Ubuntu 22.04 Removed stale test file that has linting errors Removed event loops Updated Python dependencies Change-Id: I9462b0d67ea6b5bd4c869b5f2bc8d6c57d78660c Signed-off-by: Mark Beierl --- Dockerfile | 14 +++--- {osm_nbi/tests => attic}/run_test.py | 0 osm_nbi/notifications.py | 15 +++---- osm_nbi/subscriptions.py | 47 +++++++------------ pyangbind.patch | 46 +++++++++++++++++++ requirements-dev.txt | 14 +++--- requirements-test.txt | 16 +++---- requirements.txt | 67 +++++++++++----------------- tox.ini | 4 +- 9 files changed, 123 insertions(+), 100 deletions(-) rename {osm_nbi/tests => attic}/run_test.py (100%) create mode 100644 pyangbind.patch diff --git a/Dockerfile b/Dockerfile index 2a64fdc..c4c9b6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ # devops-stages/stage-build.sh # -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -34,13 +34,15 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ debhelper \ dh-python \ git \ - python3.8 \ + python3 \ python3-all \ - python3.8-dev \ - python3-setuptools + python3-dev \ + python3-setuptools \ + python3-pip \ + tox -RUN python3 -m easy_install pip==21.3.1 -RUN pip install tox==3.24.5 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget diff --git a/osm_nbi/tests/run_test.py b/attic/run_test.py similarity index 100% rename from osm_nbi/tests/run_test.py rename to attic/run_test.py diff --git a/osm_nbi/notifications.py b/osm_nbi/notifications.py index a62670b..63d4ce8 100644 --- a/osm_nbi/notifications.py +++ b/osm_nbi/notifications.py @@ -108,12 +108,12 @@ class NotificationBase: return payload async def send_notifications( - self, subscribers: list, loop: asyncio.AbstractEventLoop = None + self, + subscribers: list, ): """ Generate tasks for all notification for an event. :param subscribers: A list of subscribers who want to be notified for event. - :param loop: Event loop object. """ notifications = [] for subscriber in subscribers: @@ -154,21 +154,19 @@ class NotificationBase: if notifications: tasks = [] - async with aiohttp.ClientSession(loop=loop) as session: + async with aiohttp.ClientSession() as session: for notification in notifications: tasks.append( asyncio.ensure_future( - self.send_notification(session, notification, loop=loop), - loop=loop, + self.send_notification(session, notification), ) ) - await asyncio.gather(*tasks, loop=loop) + await asyncio.gather(*tasks) async def send_notification( self, session: aiohttp.ClientSession, notification: dict, - loop: asyncio.AbstractEventLoop = None, retry_count: int = 5, timeout: float = 5.0, ): @@ -177,7 +175,6 @@ class NotificationBase: after maximum number of reties, then notification is dropped. :param session: An aiohttp client session object to maintain http session. :param notification: A dictionary containing all necessary data to make POST request. - :param loop: Event loop object. :param retry_count: An integer specifying the maximum number of reties for a notification. :param timeout: A float representing client timeout of each HTTP request. """ @@ -226,7 +223,7 @@ class NotificationBase: notification["payload"]["subscriptionId"], backoff_delay ) ) - await asyncio.sleep(backoff_delay, loop=loop) + await asyncio.sleep(backoff_delay) # Dropping notification self.logger.debug( "Notification {} sent failed to subscriber:{}.".format( diff --git a/osm_nbi/subscriptions.py b/osm_nbi/subscriptions.py index b178e5b..846e7d3 100644 --- a/osm_nbi/subscriptions.py +++ b/osm_nbi/subscriptions.py @@ -53,7 +53,6 @@ class SubscriptionThread(threading.Thread): self.db = None self.msg = None self.engine = engine - self.loop = None self.logger = logging.getLogger("nbi.subscriptions") self.aiomain_task_admin = ( None # asyncio task for receiving admin actions from kafka bus @@ -81,41 +80,38 @@ class SubscriptionThread(threading.Thread): # created. # Before subscribe, send dummy messages await self.msg.aiowrite( - "admin", "echo", "dummy message", loop=self.loop + "admin", + "echo", + "dummy message", ) - await self.msg.aiowrite("ns", "echo", "dummy message", loop=self.loop) - await self.msg.aiowrite("nsi", "echo", "dummy message", loop=self.loop) - await self.msg.aiowrite("vnf", "echo", "dummy message", loop=self.loop) + await self.msg.aiowrite("ns", "echo", "dummy message") + await self.msg.aiowrite("nsi", "echo", "dummy message") + await self.msg.aiowrite("vnf", "echo", "dummy message") if not kafka_working: self.logger.critical("kafka is working again") kafka_working = True if not self.aiomain_task_admin: - await asyncio.sleep(10, loop=self.loop) + await asyncio.sleep(10) self.logger.debug("Starting admin subscription task") self.aiomain_task_admin = asyncio.ensure_future( self.msg.aioread( ("admin",), - loop=self.loop, group_id=False, aiocallback=self._msg_callback, ), - loop=self.loop, ) if not self.aiomain_task: - await asyncio.sleep(10, loop=self.loop) + await asyncio.sleep(10) self.logger.debug("Starting non-admin subscription task") self.aiomain_task = asyncio.ensure_future( self.msg.aioread( ("ns", "nsi", "vnf"), - loop=self.loop, aiocallback=self._msg_callback, ), - loop=self.loop, ) done, _ = await asyncio.wait( [self.aiomain_task, self.aiomain_task_admin], timeout=None, - loop=self.loop, return_when=asyncio.FIRST_COMPLETED, ) try: @@ -142,14 +138,13 @@ class SubscriptionThread(threading.Thread): "Error accessing kafka '{}'. Retrying ...".format(e) ) kafka_working = False - await asyncio.sleep(10, loop=self.loop) + await asyncio.sleep(10) def run(self): """ Start of the thread :return: None """ - self.loop = asyncio.new_event_loop() try: if not self.db: if self.config["database"]["driver"] == "mongo": @@ -166,7 +161,6 @@ class SubscriptionThread(threading.Thread): ) if not self.msg: config_msg = self.config["message"].copy() - config_msg["loop"] = self.loop if config_msg["driver"] == "local": self.msg = msglocal.MsgLocal() self.msg.connect(config_msg) @@ -187,11 +181,7 @@ class SubscriptionThread(threading.Thread): self.logger.debug("Starting") while not self.to_terminate: try: - self.loop.run_until_complete( - asyncio.ensure_future(self.start_kafka(), loop=self.loop) - ) - # except asyncio.CancelledError: - # break # if cancelled it should end, breaking loop + asyncio.run(self.start_kafka()) except Exception as e: if not self.to_terminate: self.logger.exception( @@ -200,7 +190,6 @@ class SubscriptionThread(threading.Thread): self.logger.debug("Finishing") self._stop() - self.loop.close() async def _msg_callback(self, topic, command, params): """ @@ -265,10 +254,7 @@ class SubscriptionThread(threading.Thread): # self.logger.debug(subscribers) if subscribers: asyncio.ensure_future( - self.nslcm.send_notifications( - subscribers, loop=self.loop - ), - loop=self.loop, + self.nslcm.send_notifications(subscribers), ) else: self.logger.debug( @@ -296,8 +282,7 @@ class SubscriptionThread(threading.Thread): ) if subscribers: asyncio.ensure_future( - self.vnflcm.send_notifications(subscribers, loop=self.loop), - loop=self.loop, + self.vnflcm.send_notifications(subscribers), ) elif topic == "nsi": if command == "terminated" and params["operationState"] in ( @@ -343,7 +328,7 @@ class SubscriptionThread(threading.Thread): # writing to kafka must be done with our own loop. For this reason it is not allowed Engine to do that, # but content to be written is stored at msg_to_send for msg in msg_to_send: - await self.msg.aiowrite(*msg, loop=self.loop) + await self.msg.aiowrite(*msg) except (EngineException, DbException, MsgException) as e: self.logger.error( "Error while processing topic={} command={}: {}".format( @@ -379,6 +364,8 @@ class SubscriptionThread(threading.Thread): """ self.to_terminate = True if self.aiomain_task: - self.loop.call_soon_threadsafe(self.aiomain_task.cancel) + asyncio.get_event_loop().call_soon_threadsafe(self.aiomain_task.cancel) if self.aiomain_task_admin: - self.loop.call_soon_threadsafe(self.aiomain_task_admin.cancel) + asyncio.get_event_loop().call_soon_threadsafe( + self.aiomain_task_admin.cancel + ) diff --git a/pyangbind.patch b/pyangbind.patch new file mode 100644 index 0000000..3077299 --- /dev/null +++ b/pyangbind.patch @@ -0,0 +1,46 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400 +--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400 +*************** limitations under the License. +*** 22,27 **** +--- 22,28 ---- + from __future__ import unicode_literals + + import collections ++ from six.moves import collections_abc + import copy + import uuid + from decimal import Decimal +*************** def TypedListType(*args, **kwargs): +*** 372,378 **** + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + +--- 373,379 ---- + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections_abc.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + diff --git a/requirements-dev.txt b/requirements-dev.txt index 9ef75e0..8bbb75d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,12 +20,16 @@ async-timeout==4.0.2 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # aiokafka -bitarray==2.6.2 +bitarray==2.7.3 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # pyangbind dataclasses==0.6 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +dnspython==2.3.0 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # pymongo enum34==1.1.10 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master @@ -39,13 +43,13 @@ lxml==4.9.2 # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # pyang # pyangbind -motor==1.3.1 +motor==3.1.2 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@master # via -r requirements-dev.in osm-im @ git+https://osm.etsi.org/gerrit/osm/IM.git@master # via -r requirements-dev.in -packaging==23.0 +packaging==23.1 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # aiokafka @@ -57,7 +61,7 @@ pyangbind==0.8.1 # via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master pycryptodome==3.17 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master -pymongo==3.13.0 +pymongo==4.3.3 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # motor @@ -65,7 +69,7 @@ pyyaml==5.4.1 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master -regex==2022.10.31 +regex==2023.5.5 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # pyangbind diff --git a/requirements-test.txt b/requirements-test.txt index 0c59b6e..c81781d 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -14,7 +14,7 @@ # limitations under the License. -aiohttp==3.8.3 +aiohttp==3.8.4 # via aioresponses aioresponses==0.7.4 # via -r requirements-test.in @@ -24,13 +24,13 @@ async-timeout==4.0.2 # via aiohttp asynctest==0.13.0 # via -r requirements-test.in -attrs==22.2.0 +attrs==23.1.0 # via aiohttp -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via aiohttp -coverage==7.1.0 +coverage==7.2.5 # via -r requirements-test.in -deepdiff==6.2.3 +deepdiff==6.3.0 # via -r requirements-test.in frozenlist==1.3.3 # via @@ -42,11 +42,9 @@ multidict==6.0.4 # via # aiohttp # yarl -nose2==0.12.0 +nose2==0.13.0 # via -r requirements-test.in ordered-set==4.1.0 # via deepdiff -orjson==3.8.5 - # via deepdiff -yarl==1.8.2 +yarl==1.9.2 # via aiohttp diff --git a/requirements.txt b/requirements.txt index 941e6aa..46d9b27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,21 +14,21 @@ # limitations under the License. -aiohttp==3.8.3 +aiohttp==3.8.4 # via -r requirements.in aiosignal==1.3.1 # via aiohttp async-timeout==4.0.2 # via aiohttp -attrs==22.2.0 +attrs==23.1.0 # via # aiohttp # jsonschema autocommand==2.2.2 # via jaraco-text -certifi==2022.12.7 +certifi==2023.5.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via # aiohttp # requests @@ -41,7 +41,7 @@ debtcollector==2.5.0 # oslo-config # oslo-utils # python-keystoneclient -deepdiff==6.2.3 +deepdiff==6.3.0 # via -r requirements.in frozenlist==1.3.3 # via @@ -51,41 +51,34 @@ idna==3.4 # via # requests # yarl -importlib-resources==5.10.2 - # via - # jaraco-text - # jsonschema -inflect==6.0.2 +inflect==6.0.4 # via jaraco-text iso8601==1.1.0 # via # keystoneauth1 # oslo-utils -jaraco-classes==3.2.3 - # via jaraco-collections -jaraco-collections==3.8.0 +jaraco-collections==4.1.0 # via cherrypy jaraco-context==4.3.0 # via jaraco-text -jaraco-functools==3.5.2 +jaraco-functools==3.6.0 # via # cheroot # jaraco-text # tempora -jaraco-text==3.11.0 +jaraco-text==3.11.1 # via jaraco-collections jsonschema==4.17.3 # via -r requirements.in -keystoneauth1==5.1.1 +keystoneauth1==5.1.2 # via python-keystoneclient -more-itertools==9.0.0 +more-itertools==9.1.0 # via # cheroot # cherrypy - # jaraco-classes # jaraco-functools # jaraco-text -msgpack==1.0.4 +msgpack==1.0.5 # via oslo-serialization multidict==6.0.4 # via @@ -99,24 +92,22 @@ netifaces==0.11.0 # via oslo-utils ordered-set==4.1.0 # via deepdiff -orjson==3.8.5 - # via deepdiff os-service-types==1.7.0 # via keystoneauth1 -oslo-config==9.1.0 +oslo-config==9.1.1 # via python-keystoneclient -oslo-i18n==5.1.0 +oslo-i18n==6.0.0 # via # oslo-config # oslo-utils # python-keystoneclient -oslo-serialization==5.0.0 +oslo-serialization==5.1.1 # via python-keystoneclient oslo-utils==6.1.0 # via # oslo-serialization # python-keystoneclient -packaging==23.0 +packaging==23.1 # via # oslo-utils # python-keystoneclient @@ -128,19 +119,17 @@ pbr==5.11.1 # oslo-serialization # python-keystoneclient # stevedore -pkgutil-resolve-name==1.3.10 - # via jsonschema portend==3.1.0 # via cherrypy -pydantic==1.10.4 +pydantic==1.10.7 # via inflect pyparsing==3.0.9 # via oslo-utils pyrsistent==0.19.3 # via jsonschema -python-keystoneclient==5.0.1 +python-keystoneclient==5.1.0 # via -r requirements.in -pytz==2022.7.1 +pytz==2023.3 # via # oslo-serialization # oslo-utils @@ -149,7 +138,7 @@ pyyaml==5.4.1 # via # -r requirements.in # oslo-config -requests==2.28.2 +requests==2.30.0 # via # -r requirements.in # keystoneauth1 @@ -163,27 +152,25 @@ six==1.16.0 # keystoneauth1 # python-keystoneclient # tacacs-plus -stevedore==4.1.1 +stevedore==5.0.0 # via # keystoneauth1 # oslo-config # python-keystoneclient tacacs-plus==2.6 # via -r requirements.in -tempora==5.2.1 +tempora==5.2.2 # via portend -typing-extensions==4.4.0 +typing-extensions==4.5.0 # via pydantic -urllib3==1.26.14 +urllib3==2.0.2 # via requests -wrapt==1.14.1 +wrapt==1.15.0 # via debtcollector -yarl==1.8.2 +yarl==1.9.2 # via aiohttp -zc-lockfile==2.0 +zc-lockfile==3.0.post1 # via cherrypy -zipp==3.11.0 - # via importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/tox.ini b/tox.ini index c77d921..4c8fd06 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ toxworkdir = /tmp/.tox [testenv] usedevelop = True -basepython = python3.8 +basepython = python3.10 setenv = VIRTUAL_ENV={envdir} PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -42,9 +42,11 @@ deps = {[testenv]deps} -r{toxinidir}/requirements-dev.txt -r{toxinidir}/requirements-test.txt commands = + sh -c "patch {toxworkdir}/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py < pyangbind.patch" sh -c 'rm -f nosetests.xml' coverage erase nose2 -C --coverage osm_nbi -s osm_nbi/tests + sh -c "patch -R {toxworkdir}/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py < pyangbind.patch" coverage report --omit='*tests*' coverage html -d ./cover --omit='*tests*' coverage xml -o coverage.xml --omit=*tests* -- 2.25.1