From a264b7a460b28d7454fc95fe659da46f55b0c155 Mon Sep 17 00:00:00 2001 From: Gulsum Atici Date: Tue, 9 May 2023 14:57:22 +0300 Subject: [PATCH] Ubuntu 22.04 and Python 3.10 preparation Change-Id: I87164827a8849c16b5e3a804d9673a578e5a5593 Signed-off-by: Gulsum Atici --- Dockerfile | 9 +- NG-RO/osm_ng_ro/ns_thread.py | 1 - NG-RO/osm_ng_ro/vim_admin.py | 47 +++---- .../rel14_preparation-d9e23b217298545f.yaml | 21 ++++ requirements-dev.txt | 6 +- requirements-test.txt | 6 +- requirements.txt | 119 +++++++++--------- tox.ini | 2 +- 8 files changed, 111 insertions(+), 100 deletions(-) create mode 100644 releasenotes/notes/rel14_preparation-d9e23b217298545f.yaml diff --git a/Dockerfile b/Dockerfile index f99b758d..52c2c219 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ # devops-stages/stage-build.sh # -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -37,10 +37,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ python3 \ python3-all \ python3-dev \ - python3-setuptools + python3-setuptools \ + python3-pip -RUN python3 -m easy_install pip==21.3.1 -RUN pip install tox==3.24.5 +RUN python3 -m pip install -U pip==23.1 +RUN pip install tox ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 diff --git a/NG-RO/osm_ng_ro/ns_thread.py b/NG-RO/osm_ng_ro/ns_thread.py index de16dc95..03e8b30e 100644 --- a/NG-RO/osm_ng_ro/ns_thread.py +++ b/NG-RO/osm_ng_ro/ns_thread.py @@ -2349,7 +2349,6 @@ class NsWorker(threading.Thread): new_status, db_vim_info_update = self.item2class[ task["item"] ].new(ro_task, task_index, task_depends) - # self._create_task(ro_task, task_index, task_depends, db_ro_task_update) _update_refresh(new_status) else: refresh_at = ro_task["vim_info"]["refresh_at"] diff --git a/NG-RO/osm_ng_ro/vim_admin.py b/NG-RO/osm_ng_ro/vim_admin.py index b6e34e11..21e62817 100644 --- a/NG-RO/osm_ng_ro/vim_admin.py +++ b/NG-RO/osm_ng_ro/vim_admin.py @@ -51,15 +51,13 @@ class LockRenew: self.config = config self.logger = logger self.to_terminate = False - self.loop = None self.db = None self.task_locked_time = config["global"]["task_locked_time"] self.task_relock_time = config["global"]["task_relock_time"] self.task_max_locked_time = config["global"]["task_max_locked_time"] - def start(self, db, loop): + def start(self, db): self.db = db - self.loop = loop @staticmethod def add_lock_object(database_table, database_object, thread_object): @@ -89,9 +87,7 @@ class LockRenew: async def renew_locks(self): while not self.to_terminate: if not self.renew_list: - await asyncio.sleep( - self.task_locked_time - self.task_relock_time, loop=self.loop - ) + await asyncio.sleep(self.task_locked_time - self.task_relock_time) continue lock_object = self.renew_list[0] @@ -151,7 +147,7 @@ class LockRenew: ) else: # wait until it is time to re-lock it - await asyncio.sleep(time_to_relock, loop=self.loop) + await asyncio.sleep(time_to_relock) def stop(self): # unlock all locked items @@ -268,38 +264,31 @@ class VimAdminThread(threading.Thread): self.next_check_unused_vim = now + self.TIME_CHECK_UNUSED_VIM self.engine.unload_unused_vims() - await asyncio.sleep(self.MAX_TIME_UNATTENDED, loop=self.loop) + await asyncio.sleep(self.MAX_TIME_UNATTENDED) async def aiomain(self): kafka_working = True while not self.to_terminate: try: if not self.aiomain_task_kafka: - # await self.msg.aiowrite("admin", "echo", "dummy message", loop=self.loop) for kafka_topic in self.kafka_topics: - await self.msg.aiowrite( - kafka_topic, "echo", "dummy message", loop=self.loop - ) + await self.msg.aiowrite(kafka_topic, "echo", "dummy message") kafka_working = True self.logger.debug("Starting vim_account subscription task") self.aiomain_task_kafka = asyncio.ensure_future( self.msg.aioread( self.kafka_topics, - loop=self.loop, group_id=False, aiocallback=self._msg_callback, ), - loop=self.loop, ) if not self.aiomain_task_vim: - self.aiomain_task_vim = asyncio.ensure_future( - self.vim_watcher(), loop=self.loop - ) + self.aiomain_task_vim = asyncio.ensure_future(self.vim_watcher()) if not self.aiomain_task_renew_lock: self.aiomain_task_renew_lock = asyncio.ensure_future( - self.lock_renew.renew_locks(), loop=self.loop + self.lock_renew.renew_locks() ) done, _ = await asyncio.wait( @@ -309,7 +298,6 @@ class VimAdminThread(threading.Thread): self.aiomain_task_renew_lock, ], timeout=None, - loop=self.loop, return_when=asyncio.FIRST_COMPLETED, ) @@ -346,7 +334,7 @@ class VimAdminThread(threading.Thread): ) kafka_working = False - await asyncio.sleep(10, loop=self.loop) + await asyncio.sleep(10) def run(self): """ @@ -369,11 +357,10 @@ class VimAdminThread(threading.Thread): ) ) - self.lock_renew.start(self.db, self.loop) + self.lock_renew.start(self.db) if not self.msg: config_msg = self.config["message"].copy() - config_msg["loop"] = self.loop if config_msg["driver"] == "local": self.msg = msglocal.MsgLocal() @@ -393,11 +380,7 @@ class VimAdminThread(threading.Thread): self.logger.info("Starting") while not self.to_terminate: try: - self.loop.run_until_complete( - asyncio.ensure_future(self.aiomain(), loop=self.loop) - ) - # except asyncio.CancelledError: - # break # if cancelled it should end, breaking loop + asyncio.run_coroutine_threadsafe(self.main_task(), self.loop) except Exception as e: if not self.to_terminate: self.logger.exception( @@ -408,6 +391,10 @@ class VimAdminThread(threading.Thread): self._stop() self.loop.close() + async def main_task(self): + task = asyncio.ensure_future(self.aiomain()) + await task + async def _msg_callback(self, topic, command, params): """ Callback to process a received message from kafka @@ -471,12 +458,12 @@ class VimAdminThread(threading.Thread): self.lock_renew.to_terminate = True if self.aiomain_task_kafka: - self.loop.call_soon_threadsafe(self.aiomain_task_kafka.cancel) + self.aiomain_task_kafka.cancel() if self.aiomain_task_vim: - self.loop.call_soon_threadsafe(self.aiomain_task_vim.cancel) + self.aiomain_task_vim.cancel() if self.aiomain_task_renew_lock: - self.loop.call_soon_threadsafe(self.aiomain_task_renew_lock.cancel) + self.aiomain_task_renew_lock.cancel() self.lock_renew.stop() diff --git a/releasenotes/notes/rel14_preparation-d9e23b217298545f.yaml b/releasenotes/notes/rel14_preparation-d9e23b217298545f.yaml new file mode 100644 index 00000000..d1e756f6 --- /dev/null +++ b/releasenotes/notes/rel14_preparation-d9e23b217298545f.yaml @@ -0,0 +1,21 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +upgrade: + - | + Upgrade Ubuntu from 20.04 to 22.04 and Python from 3.8 to 3.10. + diff --git a/requirements-dev.txt b/requirements-dev.txt index dc58b6a8..840f5df6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,17 +26,17 @@ kafka-python==2.0.2 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # aiokafka -motor==1.3.1 +motor==3.1.2 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@master # via -r requirements-dev.in -packaging==23.0 +packaging==23.1 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # aiokafka pycryptodome==3.17 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master -pymongo==3.13.0 +pymongo==4.3.3 # via # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master # motor diff --git a/requirements-test.txt b/requirements-test.txt index 20b69e9b..d0375539 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,9 +16,9 @@ ####################################################################################### -e RO-plugin # via -r requirements-test.in -coverage==7.1.0 +coverage==7.2.5 # via -r requirements-test.in -mock==5.0.1 +mock==5.0.2 # via -r requirements-test.in -nose2==0.12.0 +nose2==0.13.0 # via -r requirements-test.in diff --git a/requirements.txt b/requirements.txt index 2a440e0a..27050807 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ appdirs==1.4.4 # via openstacksdk atpublic==3.1.1 # via flufl-enum -attrs==22.2.0 +attrs==23.1.0 # via # cmd2 # jsonschema @@ -32,7 +32,7 @@ azure-common==1.1.28 # azure-mgmt-compute # azure-mgmt-network # azure-mgmt-resource -azure-core==1.26.2 +azure-core==1.26.4 # via # azure-identity # azure-mgmt-core @@ -41,14 +41,14 @@ azure-identity==1.12.0 # via -r RO-VIM-azure/requirements.in azure-mgmt-compute==29.1.0 # via -r RO-VIM-azure/requirements.in -azure-mgmt-core==1.3.2 +azure-mgmt-core==1.4.0 # via # azure-mgmt-compute # azure-mgmt-network # azure-mgmt-resource -azure-mgmt-network==22.2.0 +azure-mgmt-network==23.0.1 # via -r RO-VIM-azure/requirements.in -azure-mgmt-resource==22.0.0 +azure-mgmt-resource==23.0.0 # via -r RO-VIM-azure/requirements.in bcrypt==4.0.1 # via paramiko @@ -56,7 +56,7 @@ boto==2.49.0 # via -r RO-VIM-aws/requirements.in cachetools==5.3.0 # via google-auth -certifi==2022.12.7 +certifi==2023.5.7 # via # msrest # requests @@ -64,20 +64,20 @@ cffi==1.15.1 # via # cryptography # pynacl -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 # via requests cheroot==9.0.0 # via cherrypy cherrypy==18.1.2 # via -r NG-RO/requirements.in -cliff==4.1.0 +cliff==4.3.0 # via # osc-lib # python-neutronclient # python-openstackclient -cmd2==2.4.2 +cmd2==2.4.3 # via cliff -cryptography==39.0.0 +cryptography==40.0.2 # via # -r NG-RO/requirements.in # adal @@ -87,7 +87,7 @@ cryptography==39.0.0 # paramiko # pyjwt # pyopenssl -cvprac==1.2.2 +cvprac==1.3.1 # via -r RO-SDN-arista_cloudvision/requirements.in debtcollector==2.5.0 # via @@ -101,15 +101,15 @@ decorator==5.1.1 # via # dogpile-cache # openstacksdk -dogpile-cache==1.1.8 +dogpile-cache==1.2.0 # via openstacksdk flufl-enum==5.0.1 # via pyvcloud google-api-core==2.11.0 # via google-api-python-client -google-api-python-client==2.74.0 +google-api-python-client==2.86.0 # via -r RO-VIM-gcp/requirements.in -google-auth==2.16.0 +google-auth==2.17.3 # via # -r RO-VIM-gcp/requirements.in # google-api-core @@ -119,9 +119,9 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-cloud==0.34.0 # via -r RO-VIM-gcp/requirements.in -googleapis-common-protos==1.58.0 +googleapis-common-protos==1.59.0 # via google-api-core -httplib2==0.21.0 +httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 @@ -129,11 +129,11 @@ humanfriendly==10.0 # via pyvcloud idna==3.4 # via requests -importlib-metadata==6.0.0 +importlib-metadata==6.6.0 # via # -r NG-RO/requirements.in # cliff -importlib-resources==5.10.2 +importlib-resources==5.12.0 # via jsonschema ipconflict==0.5.0 # via -r RO-VIM-aws/requirements.in @@ -146,8 +146,11 @@ iso8601==1.1.0 # python-novaclient # python-openstackclient isodate==0.6.1 - # via msrest -jaraco-functools==3.5.2 + # via + # azure-mgmt-network + # azure-mgmt-resource + # msrest +jaraco-functools==3.6.0 # via # cheroot # tempora @@ -163,7 +166,7 @@ jsonpointer==2.3 # via jsonpatch jsonschema==4.17.3 # via warlock -keystoneauth1==5.1.1 +keystoneauth1==5.1.2 # via # openstacksdk # osc-lib @@ -178,25 +181,23 @@ lxml==4.9.2 # via pyvcloud markupsafe==2.1.2 # via jinja2 -more-itertools==9.0.0 +more-itertools==9.1.0 # via # cheroot # cherrypy # jaraco-functools -msal==1.20.0 +msal==1.22.0 # via # azure-identity # msal-extensions msal-extensions==1.0.0 # via azure-identity -msgpack==1.0.4 +msgpack==1.0.5 # via oslo-serialization msrest==0.7.1 # via # -r RO-VIM-azure/requirements.in # azure-mgmt-compute - # azure-mgmt-network - # azure-mgmt-resource # msrestazure msrestazure==0.6.4 # via -r RO-VIM-azure/requirements.in @@ -218,7 +219,7 @@ netifaces==0.11.0 # oslo-utils oauthlib==3.2.2 # via requests-oauthlib -openstacksdk==0.103.0 +openstacksdk==1.1.0 # via # os-client-config # osc-lib @@ -229,17 +230,17 @@ os-service-types==1.7.0 # via # keystoneauth1 # openstacksdk -osc-lib==2.6.2 +osc-lib==2.8.0 # via # python-neutronclient # python-openstackclient -oslo-config==9.1.0 +oslo-config==9.1.1 # via # oslo-log # python-keystoneclient -oslo-context==5.0.0 +oslo-context==5.1.1 # via oslo-log -oslo-i18n==5.1.0 +oslo-i18n==6.0.0 # via # osc-lib # oslo-config @@ -251,9 +252,9 @@ oslo-i18n==5.1.0 # python-neutronclient # python-novaclient # python-openstackclient -oslo-log==5.0.2 +oslo-log==5.2.0 # via python-neutronclient -oslo-serialization==5.0.0 +oslo-serialization==5.1.1 # via # oslo-log # python-keystoneclient @@ -270,11 +271,11 @@ oslo-utils==6.1.0 # python-neutronclient # python-novaclient # python-openstackclient -packaging==23.0 +packaging==23.1 # via # oslo-utils # python-keystoneclient -paramiko==3.0.0 +paramiko==3.1.0 # via # -r RO-SDN-dpb/requirements.in # -r RO-VIM-gcp/requirements.in @@ -302,7 +303,7 @@ portalocker==2.7.0 # via msal-extensions portend==3.1.0 # via cherrypy -prettytable==3.6.0 +prettytable==3.7.0 # via # -r RO-VIM-vmware/requirements.in # cliff @@ -311,21 +312,21 @@ prettytable==3.6.0 # python-novaclient progressbar==2.5 # via -r RO-VIM-vmware/requirements.in -protobuf==4.21.12 +protobuf==4.23.0 # via # google-api-core # googleapis-common-protos py-radix==0.10.0 # via ipconflict -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pygments==2.14.0 +pygments==2.15.1 # via pyvcloud pyinotify==0.9.6 # via oslo-log @@ -335,7 +336,7 @@ pyjwt[crypto]==2.6.0 # msal pynacl==1.5.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.1.1 # via python-glanceclient pyparsing==3.0.9 # via @@ -345,6 +346,8 @@ pyperclip==1.8.2 # via cmd2 pyrsistent==0.19.3 # via jsonschema +pysocks==1.7.1 + # via requests python-cinderclient==7.4.1 # via # -r RO-VIM-openstack/requirements.in @@ -353,30 +356,30 @@ python-dateutil==2.8.2 # via # adal # oslo-log -python-glanceclient==4.2.0 +python-glanceclient==4.3.0 # via -r RO-VIM-openstack/requirements.in -python-keystoneclient==5.0.1 +python-keystoneclient==5.1.0 # via # -r RO-VIM-openstack/requirements.in # python-neutronclient # python-openstackclient -python-neutronclient==8.2.1 +python-neutronclient==9.0.0 # via -r RO-VIM-openstack/requirements.in -python-novaclient==18.2.0 +python-novaclient==18.3.0 # via # -r NG-RO/requirements.in # -r RO-VIM-openstack/requirements.in # python-openstackclient -python-openstackclient==6.0.0 +python-openstackclient==6.2.0 # via -r RO-VIM-openstack/requirements.in -pytz==2022.7.1 +pytz==2023.3 # via # oslo-serialization # oslo-utils # tempora pyvcloud==19.1.1 # via -r RO-VIM-vmware/requirements.in -pyvmomi==8.0.0.1.2 +pyvmomi==8.0.1.0 # via -r RO-VIM-vmware/requirements.in pyyaml==5.4.1 # via @@ -392,7 +395,7 @@ pyyaml==5.4.1 # openstacksdk # oslo-config # pyvcloud -requests==2.28.2 +requests[socks]==2.30.0 # via # -r NG-RO/requirements.in # -r RO-SDN-arista_cloudvision/requirements.in @@ -432,7 +435,7 @@ rfc3986==2.0.0 # via oslo-config rsa==4.9 # via google-auth -simplejson==3.18.1 +simplejson==3.19.1 # via # osc-lib # python-cinderclient @@ -450,7 +453,7 @@ six==1.16.0 # python-dateutil # python-keystoneclient # pyvmomi -stevedore==4.1.1 +stevedore==5.0.0 # via # cliff # dogpile-cache @@ -461,15 +464,15 @@ stevedore==4.1.1 # python-keystoneclient # python-novaclient # python-openstackclient -tempora==5.2.1 +tempora==5.2.2 # via portend -tqdm==4.64.1 +tqdm==4.65.0 # via ipconflict -typing-extensions==4.4.0 +typing-extensions==4.5.0 # via azure-core uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.14 +urllib3==2.0.2 # via requests uuid==1.30 # via -r RO-SDN-arista_cloudvision/requirements.in @@ -479,13 +482,13 @@ wcwidth==0.2.6 # via # cmd2 # prettytable -wrapt==1.14.1 +wrapt==1.15.0 # via # debtcollector # python-glanceclient -zc-lockfile==2.0 +zc-lockfile==3.0.post1 # via cherrypy -zipp==3.11.0 +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/tox.ini b/tox.ini index 0807ad26..c971bc17 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ setenv = XDG_CACHE_HOME=/tmp/.cache [testenv] usedevelop = True -basepython = python3 +basepython = python3.10 setenv = VIRTUAL_ENV={envdir} PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt -- 2.25.1