Squashed commit of the following:
commit
fb79786bd154505ea9c7578e6247dea565ea9c41
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Thu May 18 22:21:06 2023 -0400
Wrapping Retry for Py3.10
The retrying_async library is not Python 3.10 ready, so we are providing
a 3.10 compatible callback for it to use instead of the default one
Change-Id: I15e9b64c70d4d294c9ff0c6c7048cd257f6e1b61
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit
714d8874783b507cd66a37d1dcd2f1d3ac980257
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Thu May 18 15:08:06 2023 -0400
Wrapping Retry for Py3.10
The retrying_async library is not Python 3.10 ready, so we are providing
a 3.10 compatible callback for it to use instead of the default one
Change-Id: I6e98f6d7ebc2fe134b0e3fe37d180e383044b30b
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit
2c3c146360fe5ce949a81e0e55e0e62e7f805d0b
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Mon May 15 16:17:02 2023 -0400
Python3.10/Ubuntu 22.04 part 2
Removal of loop from all methods
Change-Id: I05bfe90f82b8c8acba3172de89c7d8e0ee08402b
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit
fcbd881700fec0522c81e8b32e3a982fb3ccbd80
Author: Gabriel Cuba <gcuba@whitestack.com>
Date: Thu May 11 02:04:17 2023 -0500
Remove charset-normalizer version constraint
Change-Id: I46311f74e949270278f685c50576f5884ba96227
Signed-off-by: Gabriel Cuba <gcuba@whitestack.com>
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit
474fd958ac88b5d2275d3acbc2fabe22e5e9344f
Author: Guillermo Calvino <guillermo.calvino@canonical.com>
Date: Fri Apr 28 11:51:43 2023 +0200
Ubuntu 22.04 and Python 3.10 preparation
Change-Id: I57a4ee39c101bdab610a6964de58eaa2653d37a3
Signed-off-by: Guillermo Calvino <guillermo.calvino@canonical.com>
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit
806cd5cf9456e69a849f4231e163da4f72379c1b
Author: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Fri Mar 24 14:03:17 2023 +0100
Fix black errors
Change-Id: I58c380853485995e2c37163a958b06072cbe24ca
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
(cherry picked from commit
9831d7e8205bce462a669a8cc2b3dc1a611c924c)
Change-Id: I0c9356df8f245b68f72f4d64ba90c9811f2e5ec7
Signed-off-by: Dario Faccin <dario.faccin@canonical.com>
# devops-stages/stage-build.sh
#
-FROM ubuntu:20.04
+FROM ubuntu:22.04
ARG APT_PROXY
RUN if [ ! -z $APT_PROXY ] ; then \
python3 \
python3-all \
python3-dev \
- python3-setuptools
+ python3-setuptools \
+ python3-pip \
+ tox
-RUN python3 -m easy_install pip==21.3.1
-RUN pip install tox==3.24.5
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
else "--skip-repos",
)
try:
- asyncio.ensure_future(
+ asyncio.create_task(
self._local_async_exec(command=command, raise_exception_on_error=False)
)
- # loop = asyncio.get_event_loop()
- # loop.run_until_complete(self._local_async_exec(command=command,
- # raise_exception_on_error=False))
except Exception as e:
self.warning(
msg="helm init failed (it was already initialized): {}".format(e)
kubectl_command: str = "/usr/bin/kubectl",
juju_command: str = "/usr/bin/juju",
log: object = None,
- loop: object = None,
on_update_db=None,
):
"""
:param kubectl_command: path to kubectl executable
:param helm_command: path to helm executable
:param log: logger
- :param: loop: Asyncio loop
"""
# parent class
K8sConnector.__init__(self, db, log=log, on_update_db=on_update_db)
self.fs = fs
- self.loop = loop or asyncio.get_event_loop()
self.log.debug("Initializing K8S Juju connector")
db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri")
self._store = MotorStore(db_uri)
- self.loading_libjuju = asyncio.Lock(loop=self.loop)
+ self.loading_libjuju = asyncio.Lock()
self.uninstall_locks = {}
self.log.debug("K8S Juju connector initialized")
will_not_delete = False
if model_name not in self.uninstall_locks:
- self.uninstall_locks[model_name] = asyncio.Lock(loop=self.loop)
+ self.uninstall_locks[model_name] = asyncio.Lock()
delete_lock = self.uninstall_locks[model_name]
while delete_lock.locked():
if not self.libjuju:
async with self.loading_libjuju:
vca_connection = await get_connection(self._store)
- self.libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log)
+ self.libjuju = Libjuju(vca_connection, log=self.log)
return self.libjuju
else:
vca_connection = await get_connection(self._store, vca_id)
- return Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ return Libjuju(vca_connection, log=self.log, n2vc=self)
def _get_kubectl(self, credentials: str) -> Kubectl:
"""
V1SecretReference,
)
from kubernetes.client.rest import ApiException
+from n2vc.libjuju import retry_callback
from retrying_async import retry
attempts=10,
delay=1,
fallback=Exception("Failed getting the secret from service account"),
+ callback=retry_callback,
)
async def get_secret_data(
self, name: str, namespace: str = "kube-system"
RBAC_LABEL_KEY_NAME = "rbac-id"
+@asyncio.coroutine
+def retry_callback(attempt, exc, args, kwargs, delay=0.5, *, loop):
+ # Specifically overridden from upstream implementation so it can
+ # continue to work with Python 3.10
+ yield from asyncio.sleep(attempt * delay)
+ return retry
+
+
class Libjuju:
def __init__(
self,
vca_connection: Connection,
- loop: asyncio.AbstractEventLoop = None,
log: logging.Logger = None,
n2vc: N2VCConnector = None,
):
Constructor
:param: vca_connection: n2vc.vca.connection object
- :param: loop: Asyncio loop
:param: log: Logger
:param: n2vc: N2VC object
"""
self.n2vc = n2vc
self.vca_connection = vca_connection
- self.loop = loop or asyncio.get_event_loop()
- self.loop.set_exception_handler(self.handle_exception)
- self.creating_model = asyncio.Lock(loop=self.loop)
+ self.creating_model = asyncio.Lock()
if self.vca_connection.is_default:
self.health_check_task = self._create_health_check_task()
def _create_health_check_task(self):
- return self.loop.create_task(self.health_check())
+ return asyncio.get_event_loop().create_task(self.health_check())
async def get_controller(self, timeout: float = 60.0) -> Controller:
"""
if controller:
await controller.disconnect()
- @retry(attempts=3, delay=5, timeout=None)
+ @retry(attempts=3, delay=5, timeout=None, callback=retry_callback)
async def add_model(self, model_name: str, cloud: VcaCloud):
"""
Create model
await self.disconnect_controller(controller)
return application_configs
- @retry(attempts=3, delay=5)
+ @retry(attempts=3, delay=5, callback=retry_callback)
async def get_model(self, controller: Controller, model_name: str) -> Model:
"""
Get model from controller
await self.disconnect_model(model)
await self.disconnect_controller(controller)
- def handle_exception(self, loop, context):
- # All unhandled exceptions by libjuju are handled here.
- pass
-
async def health_check(self, interval: float = 300.0):
"""
Health check to make sure controller and controller_model connections are OK
finally:
await self.disconnect_controller(controller)
- @retry(attempts=20, delay=5, fallback=JujuLeaderUnitNotFound())
+ @retry(
+ attempts=20, delay=5, fallback=JujuLeaderUnitNotFound(), callback=retry_callback
+ )
async def _get_leader_unit(self, application: Application) -> Unit:
unit = None
for u in application.units:
coroutine_id = ""
if include_coroutine:
try:
- if asyncio.Task.current_task() is not None:
+ if asyncio.current_task() is not None:
def print_cor_name(c):
import inspect
except Exception:
pass
- coro = asyncio.Task.current_task()._coro
+ coro = asyncio.current_task()._coro
coroutine_id = "coro-{} {}()".format(
hex(id(coro))[2:], print_cor_name(coro)
)
db: object,
fs: object,
log: object,
- loop: object,
on_update_db=None,
**kwargs,
):
:param object fs: FileSystem object managing the package artifacts (repo common
FsBase)
:param object log: the logging object to log to
- :param object loop: the loop to use for asyncio (default current thread loop)
:param on_update_db: callback called when n2vc connector updates database.
Received arguments:
table: e.g. "nsrs"
# store arguments into self
self.db = db
self.fs = fs
- self.loop = loop or asyncio.get_event_loop()
self.on_update_db = on_update_db
# generate private/public key-pair
)
from n2vc.n2vc_conn import N2VCConnector
from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
-from n2vc.libjuju import Libjuju
+from n2vc.libjuju import Libjuju, retry_callback
from n2vc.store import MotorStore
from n2vc.utils import get_ee_id_components, generate_random_alfanum_string
from n2vc.vca.connection import get_connection
db: object,
fs: object,
log: object = None,
- loop: object = None,
on_update_db=None,
):
"""
:param: db: Database object from osm_common
:param: fs: Filesystem object from osm_common
:param: log: Logger
- :param: loop: Asyncio loop
:param: on_update_db: Callback function to be called for updating the database.
"""
# parent class constructor
- N2VCConnector.__init__(
- self, db=db, fs=fs, log=log, loop=loop, on_update_db=on_update_db
- )
+ N2VCConnector.__init__(self, db=db, fs=fs, log=log, on_update_db=on_update_db)
# silence websocket traffic log
logging.getLogger("websockets.protocol").setLevel(logging.INFO)
db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri")
self._store = MotorStore(db_uri)
- self.loading_libjuju = asyncio.Lock(loop=self.loop)
+ self.loading_libjuju = asyncio.Lock()
self.delete_namespace_locks = {}
self.log.info("N2VC juju connector initialized")
# In case of native_charm is being deployed, if JujuApplicationExists error happens
# it will try to add_unit
- @retry(attempts=3, delay=5, retry_exceptions=(N2VCApplicationExists,), timeout=None)
+ @retry(
+ attempts=3,
+ delay=5,
+ retry_exceptions=(N2VCApplicationExists,),
+ timeout=None,
+ callback=retry_callback,
+ )
async def install_configuration_sw(
self,
ee_id: str,
self.log.info("Deleting namespace={}".format(namespace))
will_not_delete = False
if namespace not in self.delete_namespace_locks:
- self.delete_namespace_locks[namespace] = asyncio.Lock(loop=self.loop)
+ self.delete_namespace_locks[namespace] = asyncio.Lock()
delete_lock = self.delete_namespace_locks[namespace]
while delete_lock.locked():
if not self.libjuju:
async with self.loading_libjuju:
vca_connection = await get_connection(self._store)
- self.libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log)
+ self.libjuju = Libjuju(vca_connection, log=self.log)
return self.libjuju
else:
vca_connection = await get_connection(self._store, vca_id)
- return Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ return Libjuju(vca_connection, log=self.log, n2vc=self)
def _write_ee_id_db(self, db_dict: dict, ee_id: str):
# write ee_id to database: _admin.deployed.VCA.x
:param: vca_id: VCA ID
"""
vca_connection = await get_connection(self._store, vca_id=vca_id)
- libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ libjuju = Libjuju(vca_connection, log=self.log, n2vc=self)
controller = await libjuju.get_controller()
await libjuju.disconnect_controller(controller)
# limitations under the License.
import abc
-import asyncio
import typing
from motor.motor_asyncio import AsyncIOMotorClient
class MotorStore(Store):
- def __init__(self, uri: str, loop=None):
+ def __init__(self, uri: str):
"""
Constructor
:param: uri: Connection string to connect to the database.
- :param: loop: Asyncio Loop
"""
self._client = AsyncIOMotorClient(uri)
- self.loop = loop or asyncio.get_event_loop()
self._secret_key = None
self._config = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"])
self.encryption = Encryption(
uri=uri,
config=self._config,
encoding_type="utf-8",
- loop=self.loop,
logger_name="db",
)
fs=fslocal.FsLocal(),
db=self.db,
log=None,
- loop=self.loop,
on_update_db=None,
)
self.k8s_juju_conn._store.get_vca_id.return_value = None
}
)
logging.disable(logging.CRITICAL)
- self.libjuju = Libjuju(vca_connection, self.loop)
+ self.libjuju = Libjuju(vca_connection)
self.loop.run_until_complete(self.libjuju.disconnect())
db=self.db,
fs=fslocal.FsLocal(),
log=None,
- loop=self.loop,
on_update_db=None,
)
N2VCJujuConnector.get_public_key.assert_not_called()
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
# aiokafka
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
dataclasses==0.6
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
+dnspython==2.3.0
+ # via
+ # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
+ # pymongo
kafka-python==2.0.2
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
# aiokafka
-motor==1.3.1
+motor==3.1.2
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@paas
# via -r requirements-dev.in
-packaging==23.0
+packaging==23.1
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
# aiokafka
# temporalio
pycryptodome==3.17
# via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
-pymongo==3.13.0
+pymongo==4.3.3
# via
# -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
# motor
# limitations under the License.
asynctest
-charset-normalizer<4 # Required by aiohttp in LCM
+charset-normalizer
coverage
flake8<5.0.0
mock
#######################################################################################
asynctest==0.13.0
# via -r requirements-test.in
-certifi==2022.12.7
+certifi==2023.5.7
# via requests
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
# via
# -r requirements-test.in
# requests
-coverage==7.2.1
+coverage==7.2.5
# via -r requirements-test.in
flake8==4.0.1
# via -r requirements-test.in
# via requests
mccabe==0.6.1
# via flake8
-mock==5.0.1
+mock==5.0.2
# via -r requirements-test.in
-nose2==0.12.0
+nose2==0.13.0
# via -r requirements-test.in
pycodestyle==2.8.0
# via flake8
pyflakes==2.4.0
# via flake8
-requests==2.28.2
+requests==2.30.0
# via requests-mock
requests-mock==1.10.0
# via -r requirements-test.in
six==1.16.0
# via requests-mock
-urllib3==1.26.14
+urllib3==2.0.2
# via requests
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-charset-normalizer<4 # Required by aiohttp in LCM
+charset-normalizer
+google-auth<2.18.0
juju==3.0.0
kubernetes
-motor==1.3.1
+motor
pyasn1
pyyaml==5.4.1
retrying-async
# via paramiko
cachetools==5.3.0
# via google-auth
-certifi==2022.12.7
+certifi==2023.5.7
# via
# kubernetes
# requests
# via
# cryptography
# pynacl
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
# via
# -r requirements.in
# requests
-cryptography==39.0.1
+cryptography==40.0.2
# via paramiko
-google-auth==2.16.1
- # via kubernetes
+dnspython==2.3.0
+ # via pymongo
+google-auth==2.17.3
+ # via
+ # -r requirements.in
+ # kubernetes
idna==3.4
# via requests
juju==3.0.0
# via
# juju
# theblues
-motor==1.3.1
+motor==3.1.2
# via -r requirements.in
mypy-extensions==1.0.0
# via typing-inspect
# via juju
protobuf==3.20.3
# via macaroonbakery
-pyasn1==0.4.8
+pyasn1==0.5.0
# via
# -r requirements.in
# juju
# pyasn1-modules
# rsa
-pyasn1-modules==0.2.8
+pyasn1-modules==0.3.0
# via google-auth
pycparser==2.21
# via cffi
pymacaroons==0.13.0
# via macaroonbakery
-pymongo==3.13.0
+pymongo==4.3.3
# via motor
pynacl==1.5.0
# via
# macaroonbakery
python-dateutil==2.8.2
# via kubernetes
-pytz==2022.7.1
+pytz==2023.3
# via pyrfc3339
pyyaml==5.4.1
# via
# juju
# jujubundlelib
# kubernetes
-requests==2.28.2
+requests==2.30.0
# via
# kubernetes
# macaroonbakery
# via typing-inspect
typing-inspect==0.8.0
# via juju
-urllib3==1.26.14
+urllib3==2.0.2
# via
# kubernetes
# requests
websocket-client==1.5.1
# via kubernetes
-websockets==7.0
+websockets==11.0.3
# via juju
# The following packages are considered to be unsafe in a requirements file:
[testenv]
usedevelop = True
-basepython = python3.8
+basepython = python3.10
setenv = VIRTUAL_ENV={envdir}
PYTHONDONTWRITEBYTECODE = 1
deps = -r{toxinidir}/requirements.txt
coverage report --omit='*tests*'
coverage html -d ./cover --omit='*tests*'
coverage xml -o coverage.xml --omit=*tests*
-whitelist_externals = sh
+allowlist_externals = sh
#######################################################################################
[testenv:pip-compile]
deps = pip-tools==6.6.2
skip_install = true
-whitelist_externals = bash
+allowlist_externals = bash
[
commands =
- bash -c "for file in requirements*.in ; do \
python3 setup.py --command-packages=stdeb.command sdist_dsc
sh -c 'cd deb_dist/n2vc*/ && dpkg-buildpackage -rfakeroot -uc -us'
sh -c 'rm n2vc/requirements.txt'
-whitelist_externals = sh
+allowlist_externals = sh
#######################################################################################
[flake8]
projects / osm / N2VC.git / commitdiff