Update to Python 3.10 and Ubuntu 22.04
Removed stale test file that has linting errors
Removed event loops
Updated Python dependencies
Change-Id: I9462b0d67ea6b5bd4c869b5f2bc8d6c57d78660c
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
diff --git a/Dockerfile b/Dockerfile
index 2a64fdc..c4c9b6c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,7 +21,7 @@
# devops-stages/stage-build.sh
#
-FROM ubuntu:20.04
+FROM ubuntu:22.04
ARG APT_PROXY
RUN if [ ! -z $APT_PROXY ] ; then \
@@ -34,13 +34,15 @@
debhelper \
dh-python \
git \
- python3.8 \
+ python3 \
python3-all \
- python3.8-dev \
- python3-setuptools
+ python3-dev \
+ python3-setuptools \
+ python3-pip \
+ tox
-RUN python3 -m easy_install pip==21.3.1
-RUN pip install tox==3.24.5
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget
diff --git a/osm_nbi/tests/run_test.py b/attic/run_test.py
similarity index 100%
rename from osm_nbi/tests/run_test.py
rename to attic/run_test.py
diff --git a/osm_nbi/notifications.py b/osm_nbi/notifications.py
index a62670b..63d4ce8 100644
--- a/osm_nbi/notifications.py
+++ b/osm_nbi/notifications.py
@@ -108,12 +108,12 @@
return payload
async def send_notifications(
- self, subscribers: list, loop: asyncio.AbstractEventLoop = None
+ self,
+ subscribers: list,
):
"""
Generate tasks for all notification for an event.
:param subscribers: A list of subscribers who want to be notified for event.
- :param loop: Event loop object.
"""
notifications = []
for subscriber in subscribers:
@@ -154,21 +154,19 @@
if notifications:
tasks = []
- async with aiohttp.ClientSession(loop=loop) as session:
+ async with aiohttp.ClientSession() as session:
for notification in notifications:
tasks.append(
asyncio.ensure_future(
- self.send_notification(session, notification, loop=loop),
- loop=loop,
+ self.send_notification(session, notification),
)
)
- await asyncio.gather(*tasks, loop=loop)
+ await asyncio.gather(*tasks)
async def send_notification(
self,
session: aiohttp.ClientSession,
notification: dict,
- loop: asyncio.AbstractEventLoop = None,
retry_count: int = 5,
timeout: float = 5.0,
):
@@ -177,7 +175,6 @@
after maximum number of reties, then notification is dropped.
:param session: An aiohttp client session object to maintain http session.
:param notification: A dictionary containing all necessary data to make POST request.
- :param loop: Event loop object.
:param retry_count: An integer specifying the maximum number of reties for a notification.
:param timeout: A float representing client timeout of each HTTP request.
"""
@@ -226,7 +223,7 @@
notification["payload"]["subscriptionId"], backoff_delay
)
)
- await asyncio.sleep(backoff_delay, loop=loop)
+ await asyncio.sleep(backoff_delay)
# Dropping notification
self.logger.debug(
"Notification {} sent failed to subscriber:{}.".format(
diff --git a/osm_nbi/subscriptions.py b/osm_nbi/subscriptions.py
index b178e5b..846e7d3 100644
--- a/osm_nbi/subscriptions.py
+++ b/osm_nbi/subscriptions.py
@@ -53,7 +53,6 @@
self.db = None
self.msg = None
self.engine = engine
- self.loop = None
self.logger = logging.getLogger("nbi.subscriptions")
self.aiomain_task_admin = (
None # asyncio task for receiving admin actions from kafka bus
@@ -81,41 +80,38 @@
# created.
# Before subscribe, send dummy messages
await self.msg.aiowrite(
- "admin", "echo", "dummy message", loop=self.loop
+ "admin",
+ "echo",
+ "dummy message",
)
- await self.msg.aiowrite("ns", "echo", "dummy message", loop=self.loop)
- await self.msg.aiowrite("nsi", "echo", "dummy message", loop=self.loop)
- await self.msg.aiowrite("vnf", "echo", "dummy message", loop=self.loop)
+ await self.msg.aiowrite("ns", "echo", "dummy message")
+ await self.msg.aiowrite("nsi", "echo", "dummy message")
+ await self.msg.aiowrite("vnf", "echo", "dummy message")
if not kafka_working:
self.logger.critical("kafka is working again")
kafka_working = True
if not self.aiomain_task_admin:
- await asyncio.sleep(10, loop=self.loop)
+ await asyncio.sleep(10)
self.logger.debug("Starting admin subscription task")
self.aiomain_task_admin = asyncio.ensure_future(
self.msg.aioread(
("admin",),
- loop=self.loop,
group_id=False,
aiocallback=self._msg_callback,
),
- loop=self.loop,
)
if not self.aiomain_task:
- await asyncio.sleep(10, loop=self.loop)
+ await asyncio.sleep(10)
self.logger.debug("Starting non-admin subscription task")
self.aiomain_task = asyncio.ensure_future(
self.msg.aioread(
("ns", "nsi", "vnf"),
- loop=self.loop,
aiocallback=self._msg_callback,
),
- loop=self.loop,
)
done, _ = await asyncio.wait(
[self.aiomain_task, self.aiomain_task_admin],
timeout=None,
- loop=self.loop,
return_when=asyncio.FIRST_COMPLETED,
)
try:
@@ -142,14 +138,13 @@
"Error accessing kafka '{}'. Retrying ...".format(e)
)
kafka_working = False
- await asyncio.sleep(10, loop=self.loop)
+ await asyncio.sleep(10)
def run(self):
"""
Start of the thread
:return: None
"""
- self.loop = asyncio.new_event_loop()
try:
if not self.db:
if self.config["database"]["driver"] == "mongo":
@@ -166,7 +161,6 @@
)
if not self.msg:
config_msg = self.config["message"].copy()
- config_msg["loop"] = self.loop
if config_msg["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config_msg)
@@ -187,11 +181,7 @@
self.logger.debug("Starting")
while not self.to_terminate:
try:
- self.loop.run_until_complete(
- asyncio.ensure_future(self.start_kafka(), loop=self.loop)
- )
- # except asyncio.CancelledError:
- # break # if cancelled it should end, breaking loop
+ asyncio.run(self.start_kafka())
except Exception as e:
if not self.to_terminate:
self.logger.exception(
@@ -200,7 +190,6 @@
self.logger.debug("Finishing")
self._stop()
- self.loop.close()
async def _msg_callback(self, topic, command, params):
"""
@@ -265,10 +254,7 @@
# self.logger.debug(subscribers)
if subscribers:
asyncio.ensure_future(
- self.nslcm.send_notifications(
- subscribers, loop=self.loop
- ),
- loop=self.loop,
+ self.nslcm.send_notifications(subscribers),
)
else:
self.logger.debug(
@@ -296,8 +282,7 @@
)
if subscribers:
asyncio.ensure_future(
- self.vnflcm.send_notifications(subscribers, loop=self.loop),
- loop=self.loop,
+ self.vnflcm.send_notifications(subscribers),
)
elif topic == "nsi":
if command == "terminated" and params["operationState"] in (
@@ -343,7 +328,7 @@
# writing to kafka must be done with our own loop. For this reason it is not allowed Engine to do that,
# but content to be written is stored at msg_to_send
for msg in msg_to_send:
- await self.msg.aiowrite(*msg, loop=self.loop)
+ await self.msg.aiowrite(*msg)
except (EngineException, DbException, MsgException) as e:
self.logger.error(
"Error while processing topic={} command={}: {}".format(
@@ -379,6 +364,8 @@
"""
self.to_terminate = True
if self.aiomain_task:
- self.loop.call_soon_threadsafe(self.aiomain_task.cancel)
+ asyncio.get_event_loop().call_soon_threadsafe(self.aiomain_task.cancel)
if self.aiomain_task_admin:
- self.loop.call_soon_threadsafe(self.aiomain_task_admin.cancel)
+ asyncio.get_event_loop().call_soon_threadsafe(
+ self.aiomain_task_admin.cancel
+ )
diff --git a/pyangbind.patch b/pyangbind.patch
new file mode 100644
index 0000000..3077299
--- /dev/null
+++ b/pyangbind.patch
@@ -0,0 +1,46 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400
+--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400
+*************** limitations under the License.
+*** 22,27 ****
+--- 22,28 ----
+ from __future__ import unicode_literals
+
+ import collections
++ from six.moves import collections_abc
+ import copy
+ import uuid
+ from decimal import Decimal
+*************** def TypedListType(*args, **kwargs):
+*** 372,378 ****
+ if not isinstance(allowed_type, list):
+ allowed_type = [allowed_type]
+
+! class TypedList(collections.MutableSequence):
+ _pybind_generated_by = "TypedListType"
+ _list = list()
+
+--- 373,379 ----
+ if not isinstance(allowed_type, list):
+ allowed_type = [allowed_type]
+
+! class TypedList(collections_abc.MutableSequence):
+ _pybind_generated_by = "TypedListType"
+ _list = list()
+
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 9ef75e0..8bbb75d 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -20,12 +20,16 @@
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
# aiokafka
-bitarray==2.6.2
+bitarray==2.7.3
# via
# -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
# pyangbind
dataclasses==0.6
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
+dnspython==2.3.0
+ # via
+ # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
+ # pymongo
enum34==1.1.10
# via
# -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
@@ -39,13 +43,13 @@
# -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
# pyang
# pyangbind
-motor==1.3.1
+motor==3.1.2
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@master
# via -r requirements-dev.in
osm-im @ git+https://osm.etsi.org/gerrit/osm/IM.git@master
# via -r requirements-dev.in
-packaging==23.0
+packaging==23.1
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
# aiokafka
@@ -57,7 +61,7 @@
# via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
pycryptodome==3.17
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
-pymongo==3.13.0
+pymongo==4.3.3
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
# motor
@@ -65,7 +69,7 @@
# via
# -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
-regex==2022.10.31
+regex==2023.5.5
# via
# -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
# pyangbind
diff --git a/requirements-test.txt b/requirements-test.txt
index 0c59b6e..c81781d 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -14,7 +14,7 @@
# limitations under the License.
-aiohttp==3.8.3
+aiohttp==3.8.4
# via aioresponses
aioresponses==0.7.4
# via -r requirements-test.in
@@ -24,13 +24,13 @@
# via aiohttp
asynctest==0.13.0
# via -r requirements-test.in
-attrs==22.2.0
+attrs==23.1.0
# via aiohttp
-charset-normalizer==2.1.1
+charset-normalizer==3.1.0
# via aiohttp
-coverage==7.1.0
+coverage==7.2.5
# via -r requirements-test.in
-deepdiff==6.2.3
+deepdiff==6.3.0
# via -r requirements-test.in
frozenlist==1.3.3
# via
@@ -42,11 +42,9 @@
# via
# aiohttp
# yarl
-nose2==0.12.0
+nose2==0.13.0
# via -r requirements-test.in
ordered-set==4.1.0
# via deepdiff
-orjson==3.8.5
- # via deepdiff
-yarl==1.8.2
+yarl==1.9.2
# via aiohttp
diff --git a/requirements.txt b/requirements.txt
index 941e6aa..46d9b27 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,21 +14,21 @@
# limitations under the License.
-aiohttp==3.8.3
+aiohttp==3.8.4
# via -r requirements.in
aiosignal==1.3.1
# via aiohttp
async-timeout==4.0.2
# via aiohttp
-attrs==22.2.0
+attrs==23.1.0
# via
# aiohttp
# jsonschema
autocommand==2.2.2
# via jaraco-text
-certifi==2022.12.7
+certifi==2023.5.7
# via requests
-charset-normalizer==2.1.1
+charset-normalizer==3.1.0
# via
# aiohttp
# requests
@@ -41,7 +41,7 @@
# oslo-config
# oslo-utils
# python-keystoneclient
-deepdiff==6.2.3
+deepdiff==6.3.0
# via -r requirements.in
frozenlist==1.3.3
# via
@@ -51,41 +51,34 @@
# via
# requests
# yarl
-importlib-resources==5.10.2
- # via
- # jaraco-text
- # jsonschema
-inflect==6.0.2
+inflect==6.0.4
# via jaraco-text
iso8601==1.1.0
# via
# keystoneauth1
# oslo-utils
-jaraco-classes==3.2.3
- # via jaraco-collections
-jaraco-collections==3.8.0
+jaraco-collections==4.1.0
# via cherrypy
jaraco-context==4.3.0
# via jaraco-text
-jaraco-functools==3.5.2
+jaraco-functools==3.6.0
# via
# cheroot
# jaraco-text
# tempora
-jaraco-text==3.11.0
+jaraco-text==3.11.1
# via jaraco-collections
jsonschema==4.17.3
# via -r requirements.in
-keystoneauth1==5.1.1
+keystoneauth1==5.1.2
# via python-keystoneclient
-more-itertools==9.0.0
+more-itertools==9.1.0
# via
# cheroot
# cherrypy
- # jaraco-classes
# jaraco-functools
# jaraco-text
-msgpack==1.0.4
+msgpack==1.0.5
# via oslo-serialization
multidict==6.0.4
# via
@@ -99,24 +92,22 @@
# via oslo-utils
ordered-set==4.1.0
# via deepdiff
-orjson==3.8.5
- # via deepdiff
os-service-types==1.7.0
# via keystoneauth1
-oslo-config==9.1.0
+oslo-config==9.1.1
# via python-keystoneclient
-oslo-i18n==5.1.0
+oslo-i18n==6.0.0
# via
# oslo-config
# oslo-utils
# python-keystoneclient
-oslo-serialization==5.0.0
+oslo-serialization==5.1.1
# via python-keystoneclient
oslo-utils==6.1.0
# via
# oslo-serialization
# python-keystoneclient
-packaging==23.0
+packaging==23.1
# via
# oslo-utils
# python-keystoneclient
@@ -128,19 +119,17 @@
# oslo-serialization
# python-keystoneclient
# stevedore
-pkgutil-resolve-name==1.3.10
- # via jsonschema
portend==3.1.0
# via cherrypy
-pydantic==1.10.4
+pydantic==1.10.7
# via inflect
pyparsing==3.0.9
# via oslo-utils
pyrsistent==0.19.3
# via jsonschema
-python-keystoneclient==5.0.1
+python-keystoneclient==5.1.0
# via -r requirements.in
-pytz==2022.7.1
+pytz==2023.3
# via
# oslo-serialization
# oslo-utils
@@ -149,7 +138,7 @@
# via
# -r requirements.in
# oslo-config
-requests==2.28.2
+requests==2.30.0
# via
# -r requirements.in
# keystoneauth1
@@ -163,27 +152,25 @@
# keystoneauth1
# python-keystoneclient
# tacacs-plus
-stevedore==4.1.1
+stevedore==5.0.0
# via
# keystoneauth1
# oslo-config
# python-keystoneclient
tacacs-plus==2.6
# via -r requirements.in
-tempora==5.2.1
+tempora==5.2.2
# via portend
-typing-extensions==4.4.0
+typing-extensions==4.5.0
# via pydantic
-urllib3==1.26.14
+urllib3==2.0.2
# via requests
-wrapt==1.14.1
+wrapt==1.15.0
# via debtcollector
-yarl==1.8.2
+yarl==1.9.2
# via aiohttp
-zc-lockfile==2.0
+zc-lockfile==3.0.post1
# via cherrypy
-zipp==3.11.0
- # via importlib-resources
# The following packages are considered to be unsafe in a requirements file:
# setuptools
diff --git a/tox.ini b/tox.ini
index c77d921..4c8fd06 100644
--- a/tox.ini
+++ b/tox.ini
@@ -22,7 +22,7 @@
[testenv]
usedevelop = True
-basepython = python3.8
+basepython = python3.10
setenv = VIRTUAL_ENV={envdir}
PYTHONDONTWRITEBYTECODE = 1
deps = -r{toxinidir}/requirements.txt
@@ -42,9 +42,11 @@
-r{toxinidir}/requirements-dev.txt
-r{toxinidir}/requirements-test.txt
commands =
+ sh -c "patch {toxworkdir}/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py < pyangbind.patch"
sh -c 'rm -f nosetests.xml'
coverage erase
nose2 -C --coverage osm_nbi -s osm_nbi/tests
+ sh -c "patch -R {toxworkdir}/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py < pyangbind.patch"
coverage report --omit='*tests*'
coverage html -d ./cover --omit='*tests*'
coverage xml -o coverage.xml --omit=*tests*