n2vc/tests/unit/test_k8s_helm_conn.py

   1 ##
   2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 # not use this file except in compliance with the License. You may obtain
   4 # a copy of the License at
   5 #
   6 #         http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 # License for the specific language governing permissions and limitations
  12 # under the License.
  13 #
  14 # For those usages not covered by the Apache License, Version 2.0 please
  15 # contact: alfonso.tiernosepulveda@telefonica.com
  16 ##
  17
  18 import asynctest
  19 import logging
  20
  21 from asynctest.mock import Mock
  22 from osm_common.dbmemory import DbMemory
  23 from osm_common.fslocal import FsLocal
  24 from n2vc.k8s_helm_conn import K8sHelmConnector
  25
  26 __author__ = "Isabel Lloret <illoret@indra.es>"
  27
  28
  29 class TestK8sHelmConn(asynctest.TestCase):
  30     logging.basicConfig(level=logging.DEBUG)
  31     logger = logging.getLogger(__name__)
  32     logger.setLevel(logging.DEBUG)
  33
  34     async def setUp(self):
  35         self.db = Mock(DbMemory())
  36         self.fs = asynctest.Mock(FsLocal())
  37         self.fs.path = "./tmp/"
  38         self.namespace = "testk8s"
  39         self.service_account = "osm"
  40         self.cluster_id = "helm_cluster_id"
  41         self.cluster_uuid = self.cluster_id
  42         # pass fake kubectl and helm commands to make sure it does not call actual commands
  43         K8sHelmConnector._check_file_exists = asynctest.Mock(return_value=True)
  44         K8sHelmConnector._local_async_exec = asynctest.CoroutineMock(
  45             return_value=("", 0)
  46         )
  47         cluster_dir = self.fs.path + self.cluster_id
  48         self.kube_config = self.fs.path + self.cluster_id + "/.kube/config"
  49         self.helm_home = self.fs.path + self.cluster_id + "/.helm"
  50         self.env = {
  51             "HELM_HOME": "{}/.helm".format(cluster_dir),
  52             "KUBECONFIG": "{}/.kube/config".format(cluster_dir),
  53         }
  54         self.helm_conn = K8sHelmConnector(self.fs, self.db, log=self.logger)
  55         self.logger.debug("Set up executed")
  56
  57     @asynctest.fail_on(active_handles=True)
  58     async def test_init_env(self):
  59         # TODO
  60         pass
  61
  62     @asynctest.fail_on(active_handles=True)
  63     async def test_repo_add(self):
  64         repo_name = "bitnami"
  65         repo_url = "https://charts.bitnami.com/bitnami"
  66         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
  67
  68         await self.helm_conn.repo_add(self.cluster_uuid, repo_name, repo_url)
  69
  70         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
  71         self.helm_conn.fs.reverse_sync.assert_called_once_with(
  72             from_path=self.cluster_id
  73         )
  74         self.assertEqual(
  75             self.helm_conn._local_async_exec.call_count,
  76             2,
  77             "local_async_exec expected 2 calls, called {}".format(
  78                 self.helm_conn._local_async_exec.call_count
  79             ),
  80         )
  81
  82         repo_update_command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo update"
  83         repo_add_command = (
  84             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo add {} {}"
  85         ).format(repo_name, repo_url)
  86         calls = self.helm_conn._local_async_exec.call_args_list
  87         call0_kargs = calls[0][1]
  88         self.assertEqual(
  89             call0_kargs.get("command"),
  90             repo_update_command,
  91             "Invalid repo update command: {}".format(call0_kargs.get("command")),
  92         )
  93         self.assertEqual(
  94             call0_kargs.get("env"),
  95             self.env,
  96             "Invalid env for update command: {}".format(call0_kargs.get("env")),
  97         )
  98         call1_kargs = calls[1][1]
  99         self.assertEqual(
 100             call1_kargs.get("command"),
 101             repo_add_command,
 102             "Invalid repo add command: {}".format(call1_kargs.get("command")),
 103         )
 104         self.assertEqual(
 105             call1_kargs.get("env"),
 106             self.env,
 107             "Invalid env for add command: {}".format(call1_kargs.get("env")),
 108         )
 109
 110     @asynctest.fail_on(active_handles=True)
 111     async def test_repo_list(self):
 112         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 113
 114         await self.helm_conn.repo_list(self.cluster_uuid)
 115
 116         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 117         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 118             from_path=self.cluster_id
 119         )
 120         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo list --output yaml"
 121         self.helm_conn._local_async_exec.assert_called_with(
 122             command=command, env=self.env, raise_exception_on_error=False
 123         )
 124
 125     @asynctest.fail_on(active_handles=True)
 126     async def test_repo_remove(self):
 127         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 128         repo_name = "bitnami"
 129         await self.helm_conn.repo_remove(self.cluster_uuid, repo_name)
 130
 131         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 132         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 133             from_path=self.cluster_id
 134         )
 135         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo remove {}".format(
 136             repo_name
 137         )
 138         self.helm_conn._local_async_exec.assert_called_once_with(
 139             command=command, env=self.env, raise_exception_on_error=True
 140         )
 141
 142     @asynctest.fail_on(active_handles=True)
 143     async def test_install(self):
 144         kdu_model = "stable/openldap:1.2.2"
 145         kdu_instance = "stable-openldap-0005399828"
 146         db_dict = {}
 147         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 148         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=None)
 149         self.helm_conn._store_status = asynctest.CoroutineMock()
 150         self.helm_conn.generate_kdu_instance_name = Mock(return_value=kdu_instance)
 151
 152         await self.helm_conn.install(
 153             self.cluster_uuid,
 154             kdu_model,
 155             kdu_instance,
 156             atomic=True,
 157             namespace=self.namespace,
 158             db_dict=db_dict,
 159         )
 160
 161         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 162         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 163             from_path=self.cluster_id
 164         )
 165         self.helm_conn._store_status.assert_called_with(
 166             cluster_id=self.cluster_id,
 167             kdu_instance=kdu_instance,
 168             namespace=self.namespace,
 169             db_dict=db_dict,
 170             operation="install",
 171             run_once=True,
 172             check_every=0,
 173         )
 174         command = (
 175             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm install "
 176             "--atomic --output yaml   --timeout 300 "
 177             "--name=stable-openldap-0005399828 --namespace testk8s stable/openldap "
 178             "--version 1.2.2"
 179         )
 180         self.helm_conn._local_async_exec.assert_called_once_with(
 181             command=command, env=self.env, raise_exception_on_error=False
 182         )
 183
 184     @asynctest.fail_on(active_handles=True)
 185     async def test_upgrade(self):
 186         kdu_model = "stable/openldap:1.2.3"
 187         kdu_instance = "stable-openldap-0005399828"
 188         db_dict = {}
 189         instance_info = {
 190             "chart": "openldap-1.2.2",
 191             "name": kdu_instance,
 192             "namespace": self.namespace,
 193             "revision": 1,
 194             "status": "DEPLOYED",
 195         }
 196         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 197         self.helm_conn._store_status = asynctest.CoroutineMock()
 198         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 199             return_value=instance_info
 200         )
 201
 202         await self.helm_conn.upgrade(
 203             self.cluster_uuid, kdu_instance, kdu_model, atomic=True, db_dict=db_dict
 204         )
 205         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 206         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 207             from_path=self.cluster_id
 208         )
 209         self.helm_conn._store_status.assert_called_with(
 210             cluster_id=self.cluster_id,
 211             kdu_instance=kdu_instance,
 212             namespace=self.namespace,
 213             db_dict=db_dict,
 214             operation="upgrade",
 215             run_once=True,
 216             check_every=0,
 217         )
 218         command = (
 219             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm upgrade "
 220             "--atomic --output yaml  --timeout 300 stable-openldap-0005399828 stable/openldap --version 1.2.3"
 221         )
 222         self.helm_conn._local_async_exec.assert_called_once_with(
 223             command=command, env=self.env, raise_exception_on_error=False
 224         )
 225
 226     @asynctest.fail_on(active_handles=True)
 227     async def test_scale(self):
 228         kdu_model = "stable/openldap:1.2.3"
 229         kdu_instance = "stable-openldap-0005399828"
 230         db_dict = {}
 231         instance_info = {
 232             "chart": "openldap-1.2.3",
 233             "name": kdu_instance,
 234             "namespace": self.namespace,
 235             "revision": 1,
 236             "status": "DEPLOYED",
 237         }
 238         repo_list = [
 239             {
 240                 "name": "stable",
 241                 "url": "https://kubernetes-charts.storage.googleapis.com/",
 242             }
 243         ]
 244         kdu_values = """
 245             # Default values for openldap.
 246             # This is a YAML-formatted file.
 247             # Declare variables to be passed into your templates.
 248
 249             replicaCount: 1
 250             dummy-app:
 251               replicas: 2
 252         """
 253
 254         self.helm_conn.repo_list = asynctest.CoroutineMock(return_value=repo_list)
 255         self.helm_conn.values_kdu = asynctest.CoroutineMock(return_value=kdu_values)
 256         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 257         self.helm_conn._store_status = asynctest.CoroutineMock()
 258         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 259             return_value=instance_info
 260         )
 261
 262         # TEST-1
 263         await self.helm_conn.scale(
 264             kdu_instance,
 265             2,
 266             "",
 267             kdu_model=kdu_model,
 268             cluster_uuid=self.cluster_uuid,
 269             atomic=True,
 270             db_dict=db_dict,
 271         )
 272         command = (
 273             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 274             "/usr/bin/helm upgrade --atomic --output yaml --set replicaCount=2 "
 275             "--timeout 1800s stable-openldap-0005399828 stable/openldap "
 276             "--version 1.2.3"
 277         )
 278         self.helm_conn._local_async_exec.assert_called_once_with(
 279             command=command, env=self.env, raise_exception_on_error=False
 280         )
 281
 282         # TEST-2
 283         await self.helm_conn.scale(
 284             kdu_instance,
 285             3,
 286             "dummy-app",
 287             kdu_model=kdu_model,
 288             cluster_uuid=self.cluster_uuid,
 289             atomic=True,
 290             db_dict=db_dict,
 291         )
 292         command = (
 293             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 294             "/usr/bin/helm upgrade --atomic --output yaml --set dummy-app.replicas=3 "
 295             "--timeout 1800s stable-openldap-0005399828 stable/openldap "
 296             "--version 1.2.3"
 297         )
 298         self.helm_conn._local_async_exec.assert_called_with(
 299             command=command, env=self.env, raise_exception_on_error=False
 300         )
 301         self.helm_conn.fs.reverse_sync.assert_called_with(from_path=self.cluster_id)
 302         self.helm_conn._store_status.assert_called_with(
 303             cluster_id=self.cluster_id,
 304             kdu_instance=kdu_instance,
 305             namespace=self.namespace,
 306             db_dict=db_dict,
 307             operation="scale",
 308             run_once=True,
 309             check_every=0,
 310         )
 311
 312     @asynctest.fail_on(active_handles=True)
 313     async def test_rollback(self):
 314         kdu_instance = "stable-openldap-0005399828"
 315         db_dict = {}
 316         instance_info = {
 317             "chart": "openldap-1.2.3",
 318             "name": kdu_instance,
 319             "namespace": self.namespace,
 320             "revision": 2,
 321             "status": "DEPLOYED",
 322         }
 323         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 324         self.helm_conn._store_status = asynctest.CoroutineMock()
 325         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 326             return_value=instance_info
 327         )
 328
 329         await self.helm_conn.rollback(
 330             self.cluster_uuid, kdu_instance=kdu_instance, revision=1, db_dict=db_dict
 331         )
 332         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 333         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 334             from_path=self.cluster_id
 335         )
 336         self.helm_conn._store_status.assert_called_with(
 337             cluster_id=self.cluster_id,
 338             kdu_instance=kdu_instance,
 339             namespace=self.namespace,
 340             db_dict=db_dict,
 341             operation="rollback",
 342             run_once=True,
 343             check_every=0,
 344         )
 345         command = (
 346             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 347             "/usr/bin/helm rollback stable-openldap-0005399828 1 --wait"
 348         )
 349         self.helm_conn._local_async_exec.assert_called_once_with(
 350             command=command, env=self.env, raise_exception_on_error=False
 351         )
 352
 353     @asynctest.fail_on(active_handles=True)
 354     async def test_uninstall(self):
 355         kdu_instance = "stable-openldap-0005399828"
 356         instance_info = {
 357             "chart": "openldap-1.2.2",
 358             "name": kdu_instance,
 359             "namespace": self.namespace,
 360             "revision": 3,
 361             "status": "DEPLOYED",
 362         }
 363         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 364         self.helm_conn._store_status = asynctest.CoroutineMock()
 365         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 366             return_value=instance_info
 367         )
 368
 369         await self.helm_conn.uninstall(self.cluster_uuid, kdu_instance)
 370         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 371         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 372             from_path=self.cluster_id
 373         )
 374         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm delete --purge  {}".format(
 375             kdu_instance
 376         )
 377         self.helm_conn._local_async_exec.assert_called_once_with(
 378             command=command, env=self.env, raise_exception_on_error=True
 379         )
 380
 381     @asynctest.fail_on(active_handles=True)
 382     async def test_get_services(self):
 383         kdu_instance = "test_services_1"
 384         service = {"name": "testservice", "type": "LoadBalancer"}
 385         self.helm_conn._local_async_exec_pipe = asynctest.CoroutineMock(
 386             return_value=("", 0)
 387         )
 388         self.helm_conn._parse_services = Mock(return_value=["testservice"])
 389         self.helm_conn._get_service = asynctest.CoroutineMock(return_value=service)
 390
 391         services = await self.helm_conn.get_services(
 392             self.cluster_uuid, kdu_instance, self.namespace
 393         )
 394         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 395         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 396             from_path=self.cluster_id
 397         )
 398         self.helm_conn._parse_services.assert_called_once()
 399         command1 = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm get manifest {} ".format(
 400             kdu_instance
 401         )
 402         command2 = "/usr/bin/kubectl get --namespace={} -f -".format(self.namespace)
 403         self.helm_conn._local_async_exec_pipe.assert_called_once_with(
 404             command1, command2, env=self.env, raise_exception_on_error=True
 405         )
 406         self.assertEqual(
 407             services, [service], "Invalid service returned from get_service"
 408         )
 409
 410     @asynctest.fail_on(active_handles=True)
 411     async def test_get_service(self):
 412         service_name = "service1"
 413
 414         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 415         await self.helm_conn.get_service(
 416             self.cluster_uuid, service_name, self.namespace
 417         )
 418
 419         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 420         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 421             from_path=self.cluster_id
 422         )
 423         command = (
 424             "/usr/bin/kubectl --kubeconfig=./tmp/helm_cluster_id/.kube/config "
 425             "--namespace=testk8s get service service1 -o=yaml"
 426         )
 427         self.helm_conn._local_async_exec.assert_called_once_with(
 428             command=command, env=self.env, raise_exception_on_error=True
 429         )
 430
 431     @asynctest.fail_on(active_handles=True)
 432     async def test_inspect_kdu(self):
 433         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 434
 435         kdu_model = "stable/openldap:1.2.4"
 436         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 437         await self.helm_conn.inspect_kdu(kdu_model, repo_url)
 438
 439         command = (
 440             "/usr/bin/helm inspect  openldap --repo "
 441             "https://kubernetes-charts.storage.googleapis.com/ "
 442             "--version 1.2.4"
 443         )
 444         self.helm_conn._local_async_exec.assert_called_with(command=command)
 445
 446     @asynctest.fail_on(active_handles=True)
 447     async def test_help_kdu(self):
 448         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 449
 450         kdu_model = "stable/openldap:1.2.4"
 451         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 452         await self.helm_conn.help_kdu(kdu_model, repo_url)
 453
 454         command = (
 455             "/usr/bin/helm inspect readme openldap --repo "
 456             "https://kubernetes-charts.storage.googleapis.com/ "
 457             "--version 1.2.4"
 458         )
 459         self.helm_conn._local_async_exec.assert_called_with(command=command)
 460
 461     @asynctest.fail_on(active_handles=True)
 462     async def test_values_kdu(self):
 463         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 464
 465         kdu_model = "stable/openldap:1.2.4"
 466         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 467         await self.helm_conn.values_kdu(kdu_model, repo_url)
 468
 469         command = (
 470             "/usr/bin/helm inspect values openldap --repo "
 471             "https://kubernetes-charts.storage.googleapis.com/ "
 472             "--version 1.2.4"
 473         )
 474         self.helm_conn._local_async_exec.assert_called_with(command=command)
 475
 476     @asynctest.fail_on(active_handles=True)
 477     async def test_get_values_kdu(self):
 478         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 479
 480         kdu_instance = "stable-openldap-0005399828"
 481         await self.helm_conn.get_values_kdu(
 482             kdu_instance, self.namespace, self.env["KUBECONFIG"]
 483         )
 484
 485         command = (
 486             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm get values "
 487             "stable-openldap-0005399828 --output yaml"
 488         )
 489         self.helm_conn._local_async_exec.assert_called_with(command=command)
 490
 491     @asynctest.fail_on(active_handles=True)
 492     async def test_instances_list(self):
 493         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 494
 495         await self.helm_conn.instances_list(self.cluster_uuid)
 496         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 497         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 498             from_path=self.cluster_id
 499         )
 500         command = "/usr/bin/helm list --output yaml"
 501         self.helm_conn._local_async_exec.assert_called_once_with(
 502             command=command, env=self.env, raise_exception_on_error=True
 503         )
 504
 505     @asynctest.fail_on(active_handles=True)
 506     async def test_status_kdu(self):
 507         kdu_instance = "stable-openldap-0005399828"
 508         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 509
 510         await self.helm_conn._status_kdu(
 511             self.cluster_id, kdu_instance, self.namespace, yaml_format=True
 512         )
 513         command = (
 514             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm status {} --output yaml"
 515         ).format(kdu_instance)
 516         self.helm_conn._local_async_exec.assert_called_once_with(
 517             command=command,
 518             env=self.env,
 519             raise_exception_on_error=True,
 520             show_error_log=False,
 521         )
 522
 523     @asynctest.fail_on(active_handles=True)
 524     async def test_store_status(self):
 525         kdu_instance = "stable-openldap-0005399828"
 526         db_dict = {}
 527         status = {
 528             "info": {
 529                 "description": "Install complete",
 530                 "status": {
 531                     "code": "1",
 532                     "notes": "The openldap helm chart has been installed",
 533                 },
 534             }
 535         }
 536         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=status)
 537         self.helm_conn.write_app_status_to_db = asynctest.CoroutineMock(
 538             return_value=status
 539         )
 540
 541         await self.helm_conn._store_status(
 542             cluster_id=self.cluster_id,
 543             kdu_instance=kdu_instance,
 544             namespace=self.namespace,
 545             db_dict=db_dict,
 546             operation="install",
 547             run_once=True,
 548             check_every=0,
 549         )
 550         self.helm_conn._status_kdu.assert_called_once_with(
 551             cluster_id=self.cluster_id,
 552             kdu_instance=kdu_instance,
 553             namespace=self.namespace,
 554             yaml_format=False,
 555         )
 556         self.helm_conn.write_app_status_to_db.assert_called_once_with(
 557             db_dict=db_dict,
 558             status="Install complete",
 559             detailed_status=str(status),
 560             operation="install",
 561         )
 562
 563     @asynctest.fail_on(active_handles=True)
 564     async def test_reset_uninstall_false(self):
 565         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 566
 567         await self.helm_conn.reset(self.cluster_uuid, force=False, uninstall_sw=False)
 568         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 569         self.helm_conn.fs.file_delete.assert_called_once_with(
 570             self.cluster_id, ignore_non_exist=True
 571         )
 572         self.helm_conn._uninstall_sw.assert_not_called()
 573
 574     @asynctest.fail_on(active_handles=True)
 575     async def test_reset_uninstall(self):
 576         kdu_instance = "stable-openldap-0021099429"
 577         instances = [
 578             {
 579                 "app_version": "2.4.48",
 580                 "chart": "openldap-1.2.3",
 581                 "name": kdu_instance,
 582                 "namespace": self.namespace,
 583                 "revision": "1",
 584                 "status": "deployed",
 585                 "updated": "2020-10-30 11:11:20.376744191 +0000 UTC",
 586             }
 587         ]
 588         self.helm_conn._get_namespace = Mock(return_value=self.namespace)
 589         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 590         self.helm_conn.instances_list = asynctest.CoroutineMock(return_value=instances)
 591         self.helm_conn.uninstall = asynctest.CoroutineMock()
 592
 593         await self.helm_conn.reset(self.cluster_uuid, force=True, uninstall_sw=True)
 594         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 595         self.helm_conn.fs.file_delete.assert_called_once_with(
 596             self.cluster_id, ignore_non_exist=True
 597         )
 598         self.helm_conn._get_namespace.assert_called_once_with(
 599             cluster_uuid=self.cluster_uuid
 600         )
 601         self.helm_conn.instances_list.assert_called_once_with(
 602             cluster_uuid=self.cluster_uuid
 603         )
 604         self.helm_conn.uninstall.assert_called_once_with(
 605             cluster_uuid=self.cluster_uuid, kdu_instance=kdu_instance
 606         )
 607         self.helm_conn._uninstall_sw.assert_called_once_with(
 608             cluster_id=self.cluster_id, namespace=self.namespace
 609         )
 610
 611     @asynctest.fail_on(active_handles=True)
 612     async def test_uninstall_sw_namespace(self):
 613         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 614
 615         await self.helm_conn._uninstall_sw(self.cluster_id, self.namespace)
 616         calls = self.helm_conn._local_async_exec.call_args_list
 617         self.assertEqual(
 618             len(calls), 3, "To uninstall should have executed three commands"
 619         )
 620         call0_kargs = calls[0][1]
 621         command_0 = "/usr/bin/helm --kubeconfig={} --home={} reset".format(
 622             self.kube_config, self.helm_home
 623         )
 624         self.assertEqual(
 625             call0_kargs,
 626             {"command": command_0, "raise_exception_on_error": True, "env": self.env},
 627             "Invalid args for first call to local_exec",
 628         )
 629         call1_kargs = calls[1][1]
 630         command_1 = (
 631             "/usr/bin/kubectl --kubeconfig={} delete "
 632             "clusterrolebinding.rbac.authorization.k8s.io/osm-tiller-cluster-rule".format(
 633                 self.kube_config
 634             )
 635         )
 636         self.assertEqual(
 637             call1_kargs,
 638             {"command": command_1, "raise_exception_on_error": False, "env": self.env},
 639             "Invalid args for second call to local_exec",
 640         )
 641         call2_kargs = calls[2][1]
 642         command_2 = (
 643             "/usr/bin/kubectl --kubeconfig={} --namespace {} delete "
 644             "serviceaccount/{}".format(
 645                 self.kube_config, self.namespace, self.service_account
 646             )
 647         )
 648         self.assertEqual(
 649             call2_kargs,
 650             {"command": command_2, "raise_exception_on_error": False, "env": self.env},
 651             "Invalid args for third call to local_exec",
 652         )