From 2a3ffde1771ec4431eef96f4908b3572a883ef01 Mon Sep 17 00:00:00 2001 From: aktas Date: Thu, 24 Jun 2021 11:37:11 +0300 Subject: [PATCH] Fix Bug 1575 This fixes the race condtion when k8s cluster responding too fast Change-Id: I3f9e18c7bba942689e4b056ead60349fdb72c795 Signed-off-by: aktas --- n2vc/k8s_helm3_conn.py | 25 +++++++++++++++++--- n2vc/k8s_helm_base_conn.py | 4 ++-- n2vc/loggable.py | 2 +- n2vc/tests/unit/test_k8s_helm3_conn.py | 32 +++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/n2vc/k8s_helm3_conn.py b/n2vc/k8s_helm3_conn.py index 404485f..5bbd39b 100644 --- a/n2vc/k8s_helm3_conn.py +++ b/n2vc/k8s_helm3_conn.py @@ -110,9 +110,19 @@ class K8sHelm3Connector(K8sHelmBaseConnector): # for helm3 if namespace does not exist must create it if namespace and namespace != "kube-system": - namespaces = await self._get_namespaces(cluster_id) - if namespace not in namespaces: - await self._create_namespace(cluster_id, namespace) + if not await self._namespace_exists(cluster_id, namespace): + try: + await self._create_namespace(cluster_id, namespace) + except Exception as e: + if not await self._namespace_exists(cluster_id, namespace): + err_msg = ( + "namespace {} does not exist in cluster_id {} " + "error message: ".format( + namespace, e + ) + ) + self.log.error(err_msg) + raise K8sException(err_msg) await self._install_impl( cluster_id, @@ -226,6 +236,15 @@ class K8sHelm3Connector(K8sHelmBaseConnector): return paths, env + async def _namespace_exists(self, cluster_id, namespace) -> bool: + self.log.debug( + "checking if namespace {} exists cluster_id {}".format( + namespace, cluster_id + ) + ) + namespaces = await self._get_namespaces(cluster_id) + return namespace in namespaces if namespaces else False + async def _get_namespaces(self, cluster_id: str): self.log.debug("get namespaces cluster_id {}".format(cluster_id)) diff --git a/n2vc/k8s_helm_base_conn.py b/n2vc/k8s_helm_base_conn.py index 4a43ee2..ad59e8b 100644 --- a/n2vc/k8s_helm_base_conn.py +++ b/n2vc/k8s_helm_base_conn.py @@ -1471,11 +1471,11 @@ class K8sHelmBaseConnector(K8sConnector): # check embeded chart (file or dir) if chart_name.startswith("/"): # extract file or directory name - chart_name = chart_name[chart_name.rfind("/") + 1 :] + chart_name = chart_name[chart_name.rfind("/") + 1:] # check URL elif "://" in chart_name: # extract last portion of URL - chart_name = chart_name[chart_name.rfind("/") + 1 :] + chart_name = chart_name[chart_name.rfind("/") + 1:] name = "" for c in chart_name: diff --git a/n2vc/loggable.py b/n2vc/loggable.py index d588a1d..b3b98da 100644 --- a/n2vc/loggable.py +++ b/n2vc/loggable.py @@ -115,7 +115,7 @@ class Loggable: if not include_path: i = filename.rfind("/") if i > 0: - filename = filename[i + 1 :] + filename = filename[i + 1:] # datetime dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") diff --git a/n2vc/tests/unit/test_k8s_helm3_conn.py b/n2vc/tests/unit/test_k8s_helm3_conn.py index 25749af..b16ec76 100644 --- a/n2vc/tests/unit/test_k8s_helm3_conn.py +++ b/n2vc/tests/unit/test_k8s_helm3_conn.py @@ -21,7 +21,7 @@ import logging from asynctest.mock import Mock, patch from osm_common.dbmemory import DbMemory from osm_common.fslocal import FsLocal -from n2vc.k8s_helm3_conn import K8sHelm3Connector +from n2vc.k8s_helm3_conn import K8sHelm3Connector, K8sException __author__ = "Isabel Lloret " @@ -173,6 +173,7 @@ class TestK8sHelm3Conn(asynctest.TestCase): self.kdu_instance = "stable-openldap-0005399828" self.helm_conn.generate_kdu_instance_name = Mock(return_value=self.kdu_instance) self.helm_conn._get_namespaces = asynctest.CoroutineMock(return_value=[]) + self.helm_conn._namespace_exists = asynctest.CoroutineMock(side_effect=self.helm_conn._namespace_exists) self.helm_conn._create_namespace = asynctest.CoroutineMock() await self.helm_conn.install( @@ -184,6 +185,7 @@ class TestK8sHelm3Conn(asynctest.TestCase): db_dict=db_dict, ) + self.helm_conn._namespace_exists.assert_called_once() self.helm_conn._get_namespaces.assert_called_once() self.helm_conn._create_namespace.assert_called_once_with( self.cluster_id, self.namespace @@ -209,6 +211,34 @@ class TestK8sHelm3Conn(asynctest.TestCase): command=command, env=self.env, raise_exception_on_error=False ) + # Exception test if namespace could not being created for some reason + self.helm_conn._namespace_exists.return_value = False + self.helm_conn._create_namespace.side_effect = Exception() + + with self.assertRaises(K8sException): + await self.helm_conn.install( + self.cluster_uuid, + kdu_model, + self.kdu_instance, + atomic=True, + namespace=self.namespace, + db_dict=db_dict, + ) + + @asynctest.fail_on(active_handles=True) + async def test_namespace_exists(self): + self.helm_conn._get_namespaces = asynctest.CoroutineMock() + + self.helm_conn._get_namespaces.return_value = ['testk8s', 'kube-system'] + result = await self.helm_conn._namespace_exists(self.cluster_id, self.namespace) + self.helm_conn._get_namespaces.assert_called_once() + self.assertEqual(result, True) + + self.helm_conn._get_namespaces.reset_mock() + result = await self.helm_conn._namespace_exists(self.cluster_id, 'none-exists-namespace') + self.helm_conn._get_namespaces.assert_called_once() + self.assertEqual(result, False) + @asynctest.fail_on(active_handles=True) async def test_upgrade(self): kdu_model = "stable/openldap:1.2.3" -- 2.17.1