n2vc/tests/unit/test_k8s_helm3_conn.py

   1 ##
   2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 # not use this file except in compliance with the License. You may obtain
   4 # a copy of the License at
   5 #
   6 #         http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 # License for the specific language governing permissions and limitations
  12 # under the License.
  13 #
  14 # For those usages not covered by the Apache License, Version 2.0 please
  15 # contact: alfonso.tiernosepulveda@telefonica.com
  16 ##
  17
  18 import asynctest
  19 import logging
  20
  21 from asynctest.mock import Mock
  22 from osm_common.dbmemory import DbMemory
  23 from osm_common.fslocal import FsLocal
  24 from n2vc.k8s_helm3_conn import K8sHelm3Connector
  25
  26 __author__ = "Isabel Lloret <illoret@indra.es>"
  27
  28
  29 class TestK8sHelm3Conn(asynctest.TestCase):
  30     logging.basicConfig(level=logging.DEBUG)
  31     logger = logging.getLogger(__name__)
  32     logger.setLevel(logging.DEBUG)
  33
  34     async def setUp(self):
  35         self.db = Mock(DbMemory())
  36         self.fs = asynctest.Mock(FsLocal())
  37         self.fs.path = "./tmp/"
  38         self.namespace = "testk8s"
  39         self.cluster_id = "helm3_cluster_id"
  40         self.cluster_uuid = "{}:{}".format(self.namespace, self.cluster_id)
  41         # pass fake kubectl and helm commands to make sure it does not call actual commands
  42         K8sHelm3Connector._check_file_exists = asynctest.Mock(return_value=True)
  43         cluster_dir = self.fs.path + self.cluster_id
  44         self.env = {
  45             "HELM_CACHE_HOME": "{}/.cache/helm".format(cluster_dir),
  46             "HELM_CONFIG_HOME": "{}/.config/helm".format(cluster_dir),
  47             "HELM_DATA_HOME": "{}/.local/share/helm".format(cluster_dir),
  48             "KUBECONFIG": "{}/.kube/config".format(cluster_dir)
  49         }
  50         self.helm_conn = K8sHelm3Connector(self.fs, self.db,
  51                                            log=self.logger)
  52         self.logger.debug("Set up executed")
  53
  54     @asynctest.fail_on(active_handles=True)
  55     async def test_init_env(self):
  56         k8s_creds = "false_credentials_string"
  57         self.helm_conn._get_namespaces = asynctest.CoroutineMock(return_value=[])
  58         self.helm_conn._create_namespace = asynctest.CoroutineMock()
  59         self.helm_conn.repo_list = asynctest.CoroutineMock(return_value=[])
  60         self.helm_conn.repo_add = asynctest.CoroutineMock()
  61
  62         k8scluster_uuid, installed = await self.helm_conn.init_env(
  63             k8s_creds, namespace=self.namespace, reuse_cluster_uuid=self.cluster_id)
  64
  65         self.assertEqual(k8scluster_uuid, "{}:{}".format(self.namespace, self.cluster_id),
  66                          "Check cluster_uuid format: <namespace>.<cluster_id>")
  67         self.helm_conn._get_namespaces.assert_called_once_with(self.cluster_id)
  68         self.helm_conn._create_namespace.assert_called_once_with(self.cluster_id, self.namespace)
  69         self.helm_conn.repo_list.assert_called_once_with(k8scluster_uuid)
  70         self.helm_conn.repo_add.assert_called_once_with(
  71             k8scluster_uuid, "stable", "https://kubernetes-charts.storage.googleapis.com/")
  72         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
  73         self.logger.debug(f"cluster_uuid: {k8scluster_uuid}")
  74
  75     @asynctest.fail_on(active_handles=True)
  76     async def test_repo_add(self):
  77         repo_name = "bitnami"
  78         repo_url = "https://charts.bitnami.com/bitnami"
  79         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
  80
  81         await self.helm_conn.repo_add(self.cluster_uuid, repo_name, repo_url)
  82
  83         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
  84         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
  85         self.assertEqual(self.helm_conn._local_async_exec.call_count, 2,
  86                          "local_async_exec expected 2 calls, called {}".format(
  87                              self.helm_conn._local_async_exec.call_count))
  88
  89         repo_update_command = "/usr/bin/helm3 repo update"
  90         repo_add_command = "/usr/bin/helm3 repo add {} {}".format(repo_name, repo_url)
  91         calls = self.helm_conn._local_async_exec.call_args_list
  92         call0_kargs = calls[0][1]
  93         self.assertEqual(call0_kargs.get("command"), repo_update_command,
  94                          "Invalid repo update command: {}".format(call0_kargs.get("command")))
  95         self.assertEqual(call0_kargs.get("env"), self.env,
  96                          "Invalid env for update command: {}".format(call0_kargs.get("env")))
  97         call1_kargs = calls[1][1]
  98         self.assertEqual(call1_kargs.get("command"), repo_add_command,
  99                          "Invalid repo add command: {}".format(call1_kargs.get("command")))
 100         self.assertEqual(call1_kargs.get("env"), self.env,
 101                          "Invalid env for add command: {}".format(call1_kargs.get("env")))
 102
 103     @asynctest.fail_on(active_handles=True)
 104     async def test_repo_list(self):
 105
 106         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 107
 108         await self.helm_conn.repo_list(self.cluster_uuid)
 109
 110         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 111         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 112         command = "/usr/bin/helm3 repo list --output yaml"
 113         self.helm_conn._local_async_exec.assert_called_with(command=command, env=self.env,
 114                                                             raise_exception_on_error=False)
 115
 116     @asynctest.fail_on(active_handles=True)
 117     async def test_repo_remove(self):
 118
 119         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 120         repo_name = "bitnami"
 121         await self.helm_conn.repo_remove(self.cluster_uuid, repo_name)
 122
 123         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 124         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 125         command = "/usr/bin/helm3 repo remove {}".format(repo_name)
 126         self.helm_conn._local_async_exec.assert_called_with(command=command, env=self.env,
 127                                                             raise_exception_on_error=True)
 128
 129     @asynctest.fail_on(active_handles=True)
 130     async def test_install(self):
 131         kdu_model = "stable/openldap:1.2.2"
 132         kdu_instance = "stable-openldap-0005399828"
 133         db_dict = {}
 134         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 135         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=None)
 136         self.helm_conn._store_status = asynctest.CoroutineMock()
 137         self.helm_conn._generate_release_name = Mock(return_value="stable-openldap-0005399828")
 138         self.helm_conn._get_namespaces = asynctest.CoroutineMock(return_value=[])
 139         self.helm_conn._create_namespace = asynctest.CoroutineMock()
 140
 141         kdu_instance = await self.helm_conn.install(self.cluster_uuid,
 142                                                     kdu_model,
 143                                                     atomic=True,
 144                                                     namespace=self.namespace,
 145                                                     db_dict=db_dict)
 146
 147         self.helm_conn._get_namespaces.assert_called_once()
 148         self.helm_conn._create_namespace.assert_called_once_with(self.cluster_id, self.namespace)
 149         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 150         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 151         self.helm_conn._generate_release_name.assert_called_once_with("stable/openldap")
 152         self.helm_conn._status_kdu.assert_called_once_with(cluster_id=self.cluster_id,
 153                                                            kdu_instance=kdu_instance,
 154                                                            namespace=self.namespace,
 155                                                            show_error_log=False)
 156         self.helm_conn._store_status.assert_called_with(cluster_id=self.cluster_id,
 157                                                         kdu_instance=kdu_instance,
 158                                                         namespace=self.namespace,
 159                                                         db_dict=db_dict,
 160                                                         operation="install",
 161                                                         run_once=True,
 162                                                         check_every=0)
 163         command = "/usr/bin/helm3 install stable-openldap-0005399828 --atomic --output yaml   " \
 164                   "--timeout 300s --namespace testk8s stable/openldap --version 1.2.2"
 165         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 166                                                                  env=self.env,
 167                                                                  raise_exception_on_error=False)
 168
 169     @asynctest.fail_on(active_handles=True)
 170     async def test_upgrade(self):
 171         kdu_model = "stable/openldap:1.2.3"
 172         kdu_instance = "stable-openldap-0005399828"
 173         db_dict = {}
 174         instance_info = {
 175             "chart": "openldap-1.2.2",
 176             "name": kdu_instance,
 177             "namespace": self.namespace,
 178             "revision": 1,
 179             "status": "DEPLOYED"
 180         }
 181         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 182         self.helm_conn._store_status = asynctest.CoroutineMock()
 183         self.helm_conn.get_instance_info = asynctest.CoroutineMock(return_value=instance_info)
 184
 185         await self.helm_conn.upgrade(self.cluster_uuid,
 186                                      kdu_instance,
 187                                      kdu_model,
 188                                      atomic=True,
 189                                      db_dict=db_dict)
 190         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 191         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 192         self.helm_conn._store_status.assert_called_with(cluster_id=self.cluster_id,
 193                                                         kdu_instance=kdu_instance,
 194                                                         namespace=self.namespace,
 195                                                         db_dict=db_dict,
 196                                                         operation="upgrade",
 197                                                         run_once=True,
 198                                                         check_every=0)
 199         command = "/usr/bin/helm3 upgrade stable-openldap-0005399828 stable/openldap " \
 200                   "--namespace testk8s --atomic --output yaml  --timeout 300s  " \
 201                   "--version 1.2.3"
 202         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 203                                                                  env=self.env,
 204                                                                  raise_exception_on_error=False)
 205
 206     @asynctest.fail_on(active_handles=True)
 207     async def test_rollback(self):
 208         kdu_instance = "stable-openldap-0005399828"
 209         db_dict = {}
 210         instance_info = {
 211             "chart": "openldap-1.2.3",
 212             "name": kdu_instance,
 213             "namespace": self.namespace,
 214             "revision": 2,
 215             "status": "DEPLOYED"
 216         }
 217         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 218         self.helm_conn._store_status = asynctest.CoroutineMock()
 219         self.helm_conn.get_instance_info = asynctest.CoroutineMock(return_value=instance_info)
 220
 221         await self.helm_conn.rollback(self.cluster_uuid,
 222                                       kdu_instance=kdu_instance,
 223                                       revision=1,
 224                                       db_dict=db_dict)
 225         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 226         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 227         self.helm_conn._store_status.assert_called_with(cluster_id=self.cluster_id,
 228                                                         kdu_instance=kdu_instance,
 229                                                         namespace=self.namespace,
 230                                                         db_dict=db_dict,
 231                                                         operation="rollback",
 232                                                         run_once=True,
 233                                                         check_every=0)
 234         command = "/usr/bin/helm3 rollback stable-openldap-0005399828 1 --namespace=testk8s --wait"
 235         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 236                                                                  env=self.env,
 237                                                                  raise_exception_on_error=False)
 238
 239     @asynctest.fail_on(active_handles=True)
 240     async def test_uninstall(self):
 241         kdu_instance = "stable-openldap-0005399828"
 242         instance_info = {
 243             "chart": "openldap-1.2.2",
 244             "name": kdu_instance,
 245             "namespace": self.namespace,
 246             "revision": 3,
 247             "status": "DEPLOYED"
 248         }
 249         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 250         self.helm_conn._store_status = asynctest.CoroutineMock()
 251         self.helm_conn.get_instance_info = asynctest.CoroutineMock(return_value=instance_info)
 252
 253         await self.helm_conn.uninstall(self.cluster_uuid, kdu_instance)
 254         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 255         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 256         command = "/usr/bin/helm3 uninstall {} --namespace={}".format(
 257             kdu_instance, self.namespace)
 258         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 259                                                                  env=self.env,
 260                                                                  raise_exception_on_error=True)
 261
 262     @asynctest.fail_on(active_handles=True)
 263     async def test_get_services(self):
 264         kdu_instance = "test_services_1"
 265         service = {
 266             "name": "testservice",
 267             "type": "LoadBalancer"
 268         }
 269         self.helm_conn._local_async_exec_pipe = asynctest.CoroutineMock(return_value=("", 0))
 270         self.helm_conn._parse_services = Mock(return_value=["testservice"])
 271         self.helm_conn._get_service = asynctest.CoroutineMock(return_value=service)
 272
 273         services = await self.helm_conn.get_services(self.cluster_uuid, kdu_instance,
 274                                                      self.namespace)
 275         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 276         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 277         self.helm_conn._parse_services.assert_called_once()
 278         command1 = "/usr/bin/helm3 get manifest {} --namespace=testk8s".format(kdu_instance)
 279         command2 = "/usr/bin/kubectl get --namespace={} -f -".format(self.namespace)
 280         self.helm_conn._local_async_exec_pipe.assert_called_once_with(command1, command2,
 281                                                                       env=self.env,
 282                                                                       raise_exception_on_error=True)
 283         self.assertEqual(services, [service], "Invalid service returned from get_service")
 284
 285     @asynctest.fail_on(active_handles=True)
 286     async def test_get_service(self):
 287         service_name = "service1"
 288
 289         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 290         await self.helm_conn.get_service(self.cluster_uuid, service_name, self.namespace)
 291
 292         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 293         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 294         command = "/usr/bin/kubectl --kubeconfig=./tmp/helm3_cluster_id/.kube/config " \
 295                   "--namespace=testk8s get service service1 -o=yaml"
 296         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 297                                                                  env=self.env,
 298                                                                  raise_exception_on_error=True)
 299
 300     @asynctest.fail_on(active_handles=True)
 301     async def test_inspect_kdu(self):
 302         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 303
 304         kdu_model = "stable/openldap:1.2.4"
 305         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 306         await self.helm_conn.inspect_kdu(kdu_model, repo_url)
 307
 308         command = "/usr/bin/helm3 show all openldap --repo " \
 309                   "https://kubernetes-charts.storage.googleapis.com/ " \
 310                   "--version 1.2.4"
 311         self.helm_conn._local_async_exec.assert_called_with(command=command, encode_utf8=True)
 312
 313     @asynctest.fail_on(active_handles=True)
 314     async def test_help_kdu(self):
 315         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 316
 317         kdu_model = "stable/openldap:1.2.4"
 318         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 319         await self.helm_conn.help_kdu(kdu_model, repo_url)
 320
 321         command = "/usr/bin/helm3 show readme openldap --repo " \
 322                   "https://kubernetes-charts.storage.googleapis.com/ " \
 323                   "--version 1.2.4"
 324         self.helm_conn._local_async_exec.assert_called_with(command=command, encode_utf8=True)
 325
 326     @asynctest.fail_on(active_handles=True)
 327     async def test_values_kdu(self):
 328         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 329
 330         kdu_model = "stable/openldap:1.2.4"
 331         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 332         await self.helm_conn.values_kdu(kdu_model, repo_url)
 333
 334         command = "/usr/bin/helm3 show values openldap --repo " \
 335                   "https://kubernetes-charts.storage.googleapis.com/ " \
 336                   "--version 1.2.4"
 337         self.helm_conn._local_async_exec.assert_called_with(command=command, encode_utf8=True)
 338
 339     @asynctest.fail_on(active_handles=True)
 340     async def test_instances_list(self):
 341         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 342
 343         await self.helm_conn.instances_list(self.cluster_uuid)
 344         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 345         self.helm_conn.fs.reverse_sync.assert_called_once_with(from_path=self.cluster_id)
 346         command = "/usr/bin/helm3 list --all-namespaces  --output yaml"
 347         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 348                                                                  env=self.env,
 349                                                                  raise_exception_on_error=True)
 350
 351     @asynctest.fail_on(active_handles=True)
 352     async def test_status_kdu(self):
 353         kdu_instance = "stable-openldap-0005399828"
 354         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 355
 356         await self.helm_conn._status_kdu(self.cluster_id, kdu_instance,
 357                                          self.namespace, return_text=True)
 358         command = "/usr/bin/helm3 status {} --namespace={} --output yaml".format(
 359             kdu_instance, self.namespace
 360         )
 361         self.helm_conn._local_async_exec.assert_called_once_with(command=command,
 362                                                                  env=self.env,
 363                                                                  raise_exception_on_error=True,
 364                                                                  show_error_log=False)
 365
 366     @asynctest.fail_on(active_handles=True)
 367     async def test_store_status(self):
 368         kdu_instance = "stable-openldap-0005399828"
 369         db_dict = {}
 370         status = {
 371             "info": {
 372                 "description": "Install complete",
 373                 "status": {
 374                     "code": "1",
 375                     "notes": "The openldap helm chart has been installed"
 376                 }
 377             }
 378         }
 379         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=status)
 380         self.helm_conn.write_app_status_to_db = asynctest.CoroutineMock(return_value=status)
 381
 382         await self.helm_conn._store_status(cluster_id=self.cluster_id,
 383                                            kdu_instance=kdu_instance,
 384                                            namespace=self.namespace,
 385                                            db_dict=db_dict,
 386                                            operation="install",
 387                                            run_once=True,
 388                                            check_every=0)
 389         self.helm_conn._status_kdu.assert_called_once_with(cluster_id=self.cluster_id,
 390                                                            kdu_instance=kdu_instance,
 391                                                            namespace=self.namespace,
 392                                                            return_text=False)
 393         self.helm_conn.write_app_status_to_db.assert_called_once_with(db_dict=db_dict,
 394                                                                       status="Install complete",
 395                                                                       detailed_status=str(status),
 396                                                                       operation="install")
 397
 398     @asynctest.fail_on(active_handles=True)
 399     async def test_reset_uninstall_false(self):
 400         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 401
 402         await self.helm_conn.reset(self.cluster_uuid, force=False, uninstall_sw=False)
 403         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 404         self.helm_conn.fs.file_delete.assert_called_once_with(self.cluster_id,
 405                                                               ignore_non_exist=True)
 406         self.helm_conn._uninstall_sw.assert_not_called()
 407
 408     @asynctest.fail_on(active_handles=True)
 409     async def test_reset_uninstall(self):
 410         kdu_instance = 'stable-openldap-0021099429'
 411         instances = [
 412             {
 413                 'app_version': '2.4.48',
 414                 'chart': 'openldap-1.2.3',
 415                 'name': kdu_instance,
 416                 'namespace': self.namespace,
 417                 'revision': '1',
 418                 'status': 'deployed',
 419                 'updated': '2020-10-30 11:11:20.376744191 +0000 UTC'
 420             }
 421         ]
 422         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 423         self.helm_conn.instances_list = asynctest.CoroutineMock(return_value=instances)
 424         self.helm_conn.uninstall = asynctest.CoroutineMock()
 425
 426         await self.helm_conn.reset(self.cluster_uuid, force=True, uninstall_sw=True)
 427         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 428         self.helm_conn.fs.file_delete.assert_called_once_with(self.cluster_id,
 429                                                               ignore_non_exist=True)
 430         self.helm_conn.instances_list.assert_called_once_with(cluster_uuid=self.cluster_uuid)
 431         self.helm_conn.uninstall.assert_called_once_with(cluster_uuid=self.cluster_uuid,
 432                                                          kdu_instance=kdu_instance)
 433         self.helm_conn._uninstall_sw.assert_called_once_with(self.cluster_id, self.namespace)
 434
 435     @asynctest.fail_on(active_handles=True)
 436     async def test_sync_repos_add(self):
 437         repo_list = [
 438             {
 439                 "name": "stable",
 440                 "url": "https://kubernetes-charts.storage.googleapis.com/"
 441             }
 442         ]
 443         self.helm_conn.repo_list = asynctest.CoroutineMock(return_value=repo_list)
 444
 445         def get_one_result(*args, **kwargs):
 446             if args[0] == "k8sclusters":
 447                 return {
 448                     "_admin": {
 449                         "helm_chart_repos" : [
 450                             "4b5550a9-990d-4d95-8a48-1f4614d6ac9c"
 451                         ]
 452                     }
 453                 }
 454             elif args[0] == "k8srepos":
 455                 return {
 456                     "_id": "4b5550a9-990d-4d95-8a48-1f4614d6ac9c",
 457                     "type": "helm-chart",
 458                     "name": "bitnami",
 459                     "url": "https://charts.bitnami.com/bitnami"
 460                 }
 461         self.helm_conn.db.get_one = asynctest.Mock()
 462         self.helm_conn.db.get_one.side_effect = get_one_result
 463
 464         self.helm_conn.repo_add = asynctest.CoroutineMock()
 465         self.helm_conn.repo_remove = asynctest.CoroutineMock()
 466
 467         deleted_repo_list, added_repo_dict = await self.helm_conn.synchronize_repos(
 468             self.cluster_uuid)
 469         self.helm_conn.repo_remove.assert_not_called()
 470         self.helm_conn.repo_add.assert_called_once_with(self.cluster_uuid, "bitnami",
 471                                                         "https://charts.bitnami.com/bitnami")
 472         self.assertEqual(deleted_repo_list, [], "Deleted repo list should be empty")
 473         self.assertEqual(added_repo_dict,
 474                          {"4b5550a9-990d-4d95-8a48-1f4614d6ac9c": "bitnami"},
 475                          "Repos added should include only one bitnami")
 476
 477     @asynctest.fail_on(active_handles=True)
 478     async def test_sync_repos_delete(self):
 479         repo_list = [
 480             {
 481                 "name": "stable",
 482                 "url": "https://kubernetes-charts.storage.googleapis.com/"
 483             },
 484             {
 485                 "name": "bitnami",
 486                 "url": "https://charts.bitnami.com/bitnami"
 487             }
 488         ]
 489         self.helm_conn.repo_list = asynctest.CoroutineMock(return_value=repo_list)
 490
 491         def get_one_result(*args, **kwargs):
 492             if args[0] == "k8sclusters":
 493                 return {
 494                     "_admin": {
 495                         "helm_chart_repos": []
 496                     }
 497                 }
 498
 499         self.helm_conn.db.get_one = asynctest.Mock()
 500         self.helm_conn.db.get_one.side_effect = get_one_result
 501
 502         self.helm_conn.repo_add = asynctest.CoroutineMock()
 503         self.helm_conn.repo_remove = asynctest.CoroutineMock()
 504
 505         deleted_repo_list, added_repo_dict = await self.helm_conn.synchronize_repos(
 506             self.cluster_uuid)
 507         self.helm_conn.repo_add.assert_not_called()
 508         self.helm_conn.repo_remove.assert_called_once_with(self.cluster_uuid, "bitnami")
 509         self.assertEqual(deleted_repo_list, ["bitnami"], "Deleted repo list should be bitnami")
 510         self.assertEqual(added_repo_dict, {}, "No repos should be added")