Update from master
Squashed commit of the following:
commit fb79786bd154505ea9c7578e6247dea565ea9c41
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Thu May 18 22:21:06 2023 -0400
Wrapping Retry for Py3.10
The retrying_async library is not Python 3.10 ready, so we are providing
a 3.10 compatible callback for it to use instead of the default one
Change-Id: I15e9b64c70d4d294c9ff0c6c7048cd257f6e1b61
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit 714d8874783b507cd66a37d1dcd2f1d3ac980257
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Thu May 18 15:08:06 2023 -0400
Wrapping Retry for Py3.10
The retrying_async library is not Python 3.10 ready, so we are providing
a 3.10 compatible callback for it to use instead of the default one
Change-Id: I6e98f6d7ebc2fe134b0e3fe37d180e383044b30b
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit 2c3c146360fe5ce949a81e0e55e0e62e7f805d0b
Author: Mark Beierl <mark.beierl@canonical.com>
Date: Mon May 15 16:17:02 2023 -0400
Python3.10/Ubuntu 22.04 part 2
Removal of loop from all methods
Change-Id: I05bfe90f82b8c8acba3172de89c7d8e0ee08402b
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit fcbd881700fec0522c81e8b32e3a982fb3ccbd80
Author: Gabriel Cuba <gcuba@whitestack.com>
Date: Thu May 11 02:04:17 2023 -0500
Remove charset-normalizer version constraint
Change-Id: I46311f74e949270278f685c50576f5884ba96227
Signed-off-by: Gabriel Cuba <gcuba@whitestack.com>
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit 474fd958ac88b5d2275d3acbc2fabe22e5e9344f
Author: Guillermo Calvino <guillermo.calvino@canonical.com>
Date: Fri Apr 28 11:51:43 2023 +0200
Ubuntu 22.04 and Python 3.10 preparation
Change-Id: I57a4ee39c101bdab610a6964de58eaa2653d37a3
Signed-off-by: Guillermo Calvino <guillermo.calvino@canonical.com>
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
commit 806cd5cf9456e69a849f4231e163da4f72379c1b
Author: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Fri Mar 24 14:03:17 2023 +0100
Fix black errors
Change-Id: I58c380853485995e2c37163a958b06072cbe24ca
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
(cherry picked from commit 9831d7e8205bce462a669a8cc2b3dc1a611c924c)
Change-Id: I0c9356df8f245b68f72f4d64ba90c9811f2e5ec7
Signed-off-by: Dario Faccin <dario.faccin@canonical.com>
diff --git a/n2vc/k8s_helm_conn.py b/n2vc/k8s_helm_conn.py
index 84879c8..bbe4c48 100644
--- a/n2vc/k8s_helm_conn.py
+++ b/n2vc/k8s_helm_conn.py
@@ -78,12 +78,9 @@
else "--skip-repos",
)
try:
- asyncio.ensure_future(
+ asyncio.create_task(
self._local_async_exec(command=command, raise_exception_on_error=False)
)
- # loop = asyncio.get_event_loop()
- # loop.run_until_complete(self._local_async_exec(command=command,
- # raise_exception_on_error=False))
except Exception as e:
self.warning(
msg="helm init failed (it was already initialized): {}".format(e)
diff --git a/n2vc/k8s_juju_conn.py b/n2vc/k8s_juju_conn.py
index babe239..c197221 100644
--- a/n2vc/k8s_juju_conn.py
+++ b/n2vc/k8s_juju_conn.py
@@ -51,7 +51,6 @@
kubectl_command: str = "/usr/bin/kubectl",
juju_command: str = "/usr/bin/juju",
log: object = None,
- loop: object = None,
on_update_db=None,
):
"""
@@ -60,19 +59,17 @@
:param kubectl_command: path to kubectl executable
:param helm_command: path to helm executable
:param log: logger
- :param: loop: Asyncio loop
"""
# parent class
K8sConnector.__init__(self, db, log=log, on_update_db=on_update_db)
self.fs = fs
- self.loop = loop or asyncio.get_event_loop()
self.log.debug("Initializing K8S Juju connector")
db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri")
self._store = MotorStore(db_uri)
- self.loading_libjuju = asyncio.Lock(loop=self.loop)
+ self.loading_libjuju = asyncio.Lock()
self.uninstall_locks = {}
self.log.debug("K8S Juju connector initialized")
@@ -509,7 +506,7 @@
will_not_delete = False
if model_name not in self.uninstall_locks:
- self.uninstall_locks[model_name] = asyncio.Lock(loop=self.loop)
+ self.uninstall_locks[model_name] = asyncio.Lock()
delete_lock = self.uninstall_locks[model_name]
while delete_lock.locked():
@@ -915,11 +912,11 @@
if not self.libjuju:
async with self.loading_libjuju:
vca_connection = await get_connection(self._store)
- self.libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log)
+ self.libjuju = Libjuju(vca_connection, log=self.log)
return self.libjuju
else:
vca_connection = await get_connection(self._store, vca_id)
- return Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ return Libjuju(vca_connection, log=self.log, n2vc=self)
def _get_kubectl(self, credentials: str) -> Kubectl:
"""
diff --git a/n2vc/kubectl.py b/n2vc/kubectl.py
index d6ca09a..3fe6b53 100644
--- a/n2vc/kubectl.py
+++ b/n2vc/kubectl.py
@@ -35,6 +35,7 @@
V1SecretReference,
)
from kubernetes.client.rest import ApiException
+from n2vc.libjuju import retry_callback
from retrying_async import retry
@@ -319,6 +320,7 @@
attempts=10,
delay=1,
fallback=Exception("Failed getting the secret from service account"),
+ callback=retry_callback,
)
async def get_secret_data(
self, name: str, namespace: str = "kube-system"
diff --git a/n2vc/libjuju.py b/n2vc/libjuju.py
index 55ca859..f36ff39 100644
--- a/n2vc/libjuju.py
+++ b/n2vc/libjuju.py
@@ -61,11 +61,18 @@
RBAC_LABEL_KEY_NAME = "rbac-id"
+@asyncio.coroutine
+def retry_callback(attempt, exc, args, kwargs, delay=0.5, *, loop):
+ # Specifically overridden from upstream implementation so it can
+ # continue to work with Python 3.10
+ yield from asyncio.sleep(attempt * delay)
+ return retry
+
+
class Libjuju:
def __init__(
self,
vca_connection: Connection,
- loop: asyncio.AbstractEventLoop = None,
log: logging.Logger = None,
n2vc: N2VCConnector = None,
):
@@ -73,7 +80,6 @@
Constructor
:param: vca_connection: n2vc.vca.connection object
- :param: loop: Asyncio loop
:param: log: Logger
:param: n2vc: N2VC object
"""
@@ -82,15 +88,13 @@
self.n2vc = n2vc
self.vca_connection = vca_connection
- self.loop = loop or asyncio.get_event_loop()
- self.loop.set_exception_handler(self.handle_exception)
- self.creating_model = asyncio.Lock(loop=self.loop)
+ self.creating_model = asyncio.Lock()
if self.vca_connection.is_default:
self.health_check_task = self._create_health_check_task()
def _create_health_check_task(self):
- return self.loop.create_task(self.health_check())
+ return asyncio.get_event_loop().create_task(self.health_check())
async def get_controller(self, timeout: float = 60.0) -> Controller:
"""
@@ -155,7 +159,7 @@
if controller:
await controller.disconnect()
- @retry(attempts=3, delay=5, timeout=None)
+ @retry(attempts=3, delay=5, timeout=None, callback=retry_callback)
async def add_model(self, model_name: str, cloud: VcaCloud):
"""
Create model
@@ -270,7 +274,7 @@
await self.disconnect_controller(controller)
return application_configs
- @retry(attempts=3, delay=5)
+ @retry(attempts=3, delay=5, callback=retry_callback)
async def get_model(self, controller: Controller, model_name: str) -> Model:
"""
Get model from controller
@@ -1640,10 +1644,6 @@
await self.disconnect_model(model)
await self.disconnect_controller(controller)
- def handle_exception(self, loop, context):
- # All unhandled exceptions by libjuju are handled here.
- pass
-
async def health_check(self, interval: float = 300.0):
"""
Health check to make sure controller and controller_model connections are OK
@@ -1848,7 +1848,9 @@
finally:
await self.disconnect_controller(controller)
- @retry(attempts=20, delay=5, fallback=JujuLeaderUnitNotFound())
+ @retry(
+ attempts=20, delay=5, fallback=JujuLeaderUnitNotFound(), callback=retry_callback
+ )
async def _get_leader_unit(self, application: Application) -> Unit:
unit = None
for u in application.units:
diff --git a/n2vc/loggable.py b/n2vc/loggable.py
index cbaa116..d129b4b 100644
--- a/n2vc/loggable.py
+++ b/n2vc/loggable.py
@@ -131,7 +131,7 @@
coroutine_id = ""
if include_coroutine:
try:
- if asyncio.Task.current_task() is not None:
+ if asyncio.current_task() is not None:
def print_cor_name(c):
import inspect
@@ -143,7 +143,7 @@
except Exception:
pass
- coro = asyncio.Task.current_task()._coro
+ coro = asyncio.current_task()._coro
coroutine_id = "coro-{} {}()".format(
hex(id(coro))[2:], print_cor_name(coro)
)
diff --git a/n2vc/n2vc_conn.py b/n2vc/n2vc_conn.py
index 5752da7..4fa7e36 100644
--- a/n2vc/n2vc_conn.py
+++ b/n2vc/n2vc_conn.py
@@ -54,7 +54,6 @@
db: object,
fs: object,
log: object,
- loop: object,
on_update_db=None,
**kwargs,
):
@@ -64,7 +63,6 @@
:param object fs: FileSystem object managing the package artifacts (repo common
FsBase)
:param object log: the logging object to log to
- :param object loop: the loop to use for asyncio (default current thread loop)
:param on_update_db: callback called when n2vc connector updates database.
Received arguments:
table: e.g. "nsrs"
@@ -85,7 +83,6 @@
# store arguments into self
self.db = db
self.fs = fs
- self.loop = loop or asyncio.get_event_loop()
self.on_update_db = on_update_db
# generate private/public key-pair
diff --git a/n2vc/n2vc_juju_conn.py b/n2vc/n2vc_juju_conn.py
index cbca396..9d0cdfa 100644
--- a/n2vc/n2vc_juju_conn.py
+++ b/n2vc/n2vc_juju_conn.py
@@ -37,7 +37,7 @@
)
from n2vc.n2vc_conn import N2VCConnector
from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
-from n2vc.libjuju import Libjuju
+from n2vc.libjuju import Libjuju, retry_callback
from n2vc.store import MotorStore
from n2vc.utils import get_ee_id_components, generate_random_alfanum_string
from n2vc.vca.connection import get_connection
@@ -61,7 +61,6 @@
db: object,
fs: object,
log: object = None,
- loop: object = None,
on_update_db=None,
):
"""
@@ -70,14 +69,11 @@
:param: db: Database object from osm_common
:param: fs: Filesystem object from osm_common
:param: log: Logger
- :param: loop: Asyncio loop
:param: on_update_db: Callback function to be called for updating the database.
"""
# parent class constructor
- N2VCConnector.__init__(
- self, db=db, fs=fs, log=log, loop=loop, on_update_db=on_update_db
- )
+ N2VCConnector.__init__(self, db=db, fs=fs, log=log, on_update_db=on_update_db)
# silence websocket traffic log
logging.getLogger("websockets.protocol").setLevel(logging.INFO)
@@ -88,7 +84,7 @@
db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri")
self._store = MotorStore(db_uri)
- self.loading_libjuju = asyncio.Lock(loop=self.loop)
+ self.loading_libjuju = asyncio.Lock()
self.delete_namespace_locks = {}
self.log.info("N2VC juju connector initialized")
@@ -359,7 +355,13 @@
# In case of native_charm is being deployed, if JujuApplicationExists error happens
# it will try to add_unit
- @retry(attempts=3, delay=5, retry_exceptions=(N2VCApplicationExists,), timeout=None)
+ @retry(
+ attempts=3,
+ delay=5,
+ retry_exceptions=(N2VCApplicationExists,),
+ timeout=None,
+ callback=retry_callback,
+ )
async def install_configuration_sw(
self,
ee_id: str,
@@ -772,7 +774,7 @@
self.log.info("Deleting namespace={}".format(namespace))
will_not_delete = False
if namespace not in self.delete_namespace_locks:
- self.delete_namespace_locks[namespace] = asyncio.Lock(loop=self.loop)
+ self.delete_namespace_locks[namespace] = asyncio.Lock()
delete_lock = self.delete_namespace_locks[namespace]
while delete_lock.locked():
@@ -1148,11 +1150,11 @@
if not self.libjuju:
async with self.loading_libjuju:
vca_connection = await get_connection(self._store)
- self.libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log)
+ self.libjuju = Libjuju(vca_connection, log=self.log)
return self.libjuju
else:
vca_connection = await get_connection(self._store, vca_id)
- return Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ return Libjuju(vca_connection, log=self.log, n2vc=self)
def _write_ee_id_db(self, db_dict: dict, ee_id: str):
# write ee_id to database: _admin.deployed.VCA.x
@@ -1536,6 +1538,6 @@
:param: vca_id: VCA ID
"""
vca_connection = await get_connection(self._store, vca_id=vca_id)
- libjuju = Libjuju(vca_connection, loop=self.loop, log=self.log, n2vc=self)
+ libjuju = Libjuju(vca_connection, log=self.log, n2vc=self)
controller = await libjuju.get_controller()
await libjuju.disconnect_controller(controller)
diff --git a/n2vc/store.py b/n2vc/store.py
index e9586d7..c8e5910 100644
--- a/n2vc/store.py
+++ b/n2vc/store.py
@@ -13,7 +13,6 @@
# limitations under the License.
import abc
-import asyncio
import typing
from motor.motor_asyncio import AsyncIOMotorClient
@@ -183,22 +182,19 @@
class MotorStore(Store):
- def __init__(self, uri: str, loop=None):
+ def __init__(self, uri: str):
"""
Constructor
:param: uri: Connection string to connect to the database.
- :param: loop: Asyncio Loop
"""
self._client = AsyncIOMotorClient(uri)
- self.loop = loop or asyncio.get_event_loop()
self._secret_key = None
self._config = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"])
self.encryption = Encryption(
uri=uri,
config=self._config,
encoding_type="utf-8",
- loop=self.loop,
logger_name="db",
)
diff --git a/n2vc/tests/unit/test_k8s_juju_conn.py b/n2vc/tests/unit/test_k8s_juju_conn.py
index 1cc0809..1de1288 100644
--- a/n2vc/tests/unit/test_k8s_juju_conn.py
+++ b/n2vc/tests/unit/test_k8s_juju_conn.py
@@ -72,7 +72,6 @@
fs=fslocal.FsLocal(),
db=self.db,
log=None,
- loop=self.loop,
on_update_db=None,
)
self.k8s_juju_conn._store.get_vca_id.return_value = None
diff --git a/n2vc/tests/unit/test_libjuju.py b/n2vc/tests/unit/test_libjuju.py
index 9f21bc6..38d8d0e 100644
--- a/n2vc/tests/unit/test_libjuju.py
+++ b/n2vc/tests/unit/test_libjuju.py
@@ -78,7 +78,7 @@
}
)
logging.disable(logging.CRITICAL)
- self.libjuju = Libjuju(vca_connection, self.loop)
+ self.libjuju = Libjuju(vca_connection)
self.loop.run_until_complete(self.libjuju.disconnect())
diff --git a/n2vc/tests/unit/test_n2vc_juju_conn.py b/n2vc/tests/unit/test_n2vc_juju_conn.py
index deb98ce..456ec1e 100644
--- a/n2vc/tests/unit/test_n2vc_juju_conn.py
+++ b/n2vc/tests/unit/test_n2vc_juju_conn.py
@@ -73,7 +73,6 @@
db=self.db,
fs=fslocal.FsLocal(),
log=None,
- loop=self.loop,
on_update_db=None,
)
N2VCJujuConnector.get_public_key.assert_not_called()