Fix Bug 1575

This fixes the race condtion when k8s cluster responding too fast

Change-Id: I3f9e18c7bba942689e4b056ead60349fdb72c795
Signed-off-by: aktas <emin.aktas@ulakhaberlesme.com.tr>

diff --git a/n2vc/k8s_helm3_conn.py b/n2vc/k8s_helm3_conn.py
index 6afadbf..68ad37e 100644 (file)
--- a/n2vc/k8s_helm3_conn.py
+++ b/n2vc/k8s_helm3_conn.py
@@ -92,9 +92,19 @@ class K8sHelm3Connector(K8sHelmBaseConnector):
 
         # for helm3 if namespace does not exist must create it
         if namespace and namespace != "kube-system":
-            namespaces = await self._get_namespaces(cluster_id)
-            if namespace not in namespaces:
-                await self._create_namespace(cluster_id, namespace)
+            if not await self._namespace_exists(cluster_id, namespace):
+                try:
+                    await self._create_namespace(cluster_id, namespace)
+                except Exception as e:
+                    if not await self._namespace_exists(cluster_id, namespace):
+                        err_msg = (
+                            "namespace {} does not exist in cluster_id {} "
+                            "error message: ".format(
+                                namespace, e
+                            )
+                        )
+                        self.log.error(err_msg)
+                        raise K8sException(err_msg)
 
         await self._install_impl(
             cluster_id,
@@ -208,8 +218,16 @@ class K8sHelm3Connector(K8sHelmBaseConnector):
 
         return paths, env
 
-    async def _get_namespaces(self,
-                              cluster_id: str):
+    async def _namespace_exists(self, cluster_id, namespace) -> bool:
+        self.log.debug(
+            "checking if namespace {} exists cluster_id {}".format(
+                namespace, cluster_id
+            )
+        )
+        namespaces = await self._get_namespaces(cluster_id)
+        return namespace in namespaces if namespaces else False
+
+    async def _get_namespaces(self, cluster_id: str):
 
         self.log.debug("get namespaces cluster_id {}".format(cluster_id))
 

diff --git a/n2vc/tests/unit/test_k8s_helm3_conn.py b/n2vc/tests/unit/test_k8s_helm3_conn.py
index 3b81323..5f43804 100644 (file)
--- a/n2vc/tests/unit/test_k8s_helm3_conn.py
+++ b/n2vc/tests/unit/test_k8s_helm3_conn.py
@@ -21,7 +21,7 @@ import logging
 from asynctest.mock import Mock
 from osm_common.dbmemory import DbMemory
 from osm_common.fslocal import FsLocal
-from n2vc.k8s_helm3_conn import K8sHelm3Connector
+from n2vc.k8s_helm3_conn import K8sHelm3Connector, K8sException
 
 __author__ = "Isabel Lloret <illoret@indra.es>"
 
@@ -137,6 +137,7 @@ class TestK8sHelm3Conn(asynctest.TestCase):
         self.kdu_instance = "stable-openldap-0005399828"
         self.helm_conn.generate_kdu_instance_name = Mock(return_value=self.kdu_instance)
         self.helm_conn._get_namespaces = asynctest.CoroutineMock(return_value=[])
+        self.helm_conn._namespace_exists = asynctest.CoroutineMock(side_effect=self.helm_conn._namespace_exists)
         self.helm_conn._create_namespace = asynctest.CoroutineMock()
 
         await self.helm_conn.install(
@@ -148,6 +149,7 @@ class TestK8sHelm3Conn(asynctest.TestCase):
             db_dict=db_dict
         )
 
+        self.helm_conn._namespace_exists.assert_called_once()
         self.helm_conn._get_namespaces.assert_called_once()
         self.helm_conn._create_namespace.assert_called_once_with(self.cluster_id, self.namespace)
         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
@@ -165,6 +167,34 @@ class TestK8sHelm3Conn(asynctest.TestCase):
                                                                  env=self.env,
                                                                  raise_exception_on_error=False)
 
+        # Exception test if namespace could not being created for some reason
+        self.helm_conn._namespace_exists.return_value = False
+        self.helm_conn._create_namespace.side_effect = Exception()
+
+        with self.assertRaises(K8sException):
+            await self.helm_conn.install(
+                self.cluster_uuid,
+                kdu_model,
+                self.kdu_instance,
+                atomic=True,
+                namespace=self.namespace,
+                db_dict=db_dict,
+            )
+
+    @asynctest.fail_on(active_handles=True)
+    async def test_namespace_exists(self):
+        self.helm_conn._get_namespaces = asynctest.CoroutineMock()
+
+        self.helm_conn._get_namespaces.return_value = ['testk8s', 'kube-system']
+        result = await self.helm_conn._namespace_exists(self.cluster_id, self.namespace)
+        self.helm_conn._get_namespaces.assert_called_once()
+        self.assertEqual(result, True)
+
+        self.helm_conn._get_namespaces.reset_mock()
+        result = await self.helm_conn._namespace_exists(self.cluster_id, 'none-exists-namespace')
+        self.helm_conn._get_namespaces.assert_called_once()
+        self.assertEqual(result, False)
+
     @asynctest.fail_on(active_handles=True)
     async def test_upgrade(self):
         kdu_model = "stable/openldap:1.2.3"