n2vc/tests/unit/test_k8s_helm_conn.py

   1 ##
   2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 # not use this file except in compliance with the License. You may obtain
   4 # a copy of the License at
   5 #
   6 #         http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 # License for the specific language governing permissions and limitations
  12 # under the License.
  13 #
  14 # For those usages not covered by the Apache License, Version 2.0 please
  15 # contact: alfonso.tiernosepulveda@telefonica.com
  16 ##
  17
  18 import asynctest
  19 import logging
  20
  21 from asynctest.mock import Mock
  22 from osm_common.dbmemory import DbMemory
  23 from osm_common.fslocal import FsLocal
  24 from n2vc.k8s_helm_conn import K8sHelmConnector
  25
  26 __author__ = "Isabel Lloret <illoret@indra.es>"
  27
  28
  29 class TestK8sHelmConn(asynctest.TestCase):
  30     logging.basicConfig(level=logging.DEBUG)
  31     logger = logging.getLogger(__name__)
  32     logger.setLevel(logging.DEBUG)
  33
  34     async def setUp(self):
  35         self.db = Mock(DbMemory())
  36         self.fs = asynctest.Mock(FsLocal())
  37         self.fs.path = "./tmp/"
  38         self.namespace = "testk8s"
  39         self.service_account = "osm"
  40         self.cluster_id = "helm_cluster_id"
  41         self.cluster_uuid = self.cluster_id
  42         # pass fake kubectl and helm commands to make sure it does not call actual commands
  43         K8sHelmConnector._check_file_exists = asynctest.Mock(return_value=True)
  44         K8sHelmConnector._local_async_exec = asynctest.CoroutineMock(
  45             return_value=(0, "")
  46         )
  47         cluster_dir = self.fs.path + self.cluster_id
  48         self.kube_config = self.fs.path + self.cluster_id + "/.kube/config"
  49         self.helm_home = self.fs.path + self.cluster_id + "/.helm"
  50         self.env = {
  51             "HELM_HOME": "{}/.helm".format(cluster_dir),
  52             "KUBECONFIG": "{}/.kube/config".format(cluster_dir),
  53         }
  54         self.helm_conn = K8sHelmConnector(self.fs, self.db, log=self.logger)
  55         self.logger.debug("Set up executed")
  56
  57     @asynctest.fail_on(active_handles=True)
  58     async def test_init_env(self):
  59         # TODO
  60         pass
  61
  62     @asynctest.fail_on(active_handles=True)
  63     async def test_repo_add(self):
  64         repo_name = "bitnami"
  65         repo_url = "https://charts.bitnami.com/bitnami"
  66         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
  67
  68         await self.helm_conn.repo_add(self.cluster_uuid, repo_name, repo_url)
  69
  70         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
  71         self.helm_conn.fs.reverse_sync.assert_called_once_with(
  72             from_path=self.cluster_id
  73         )
  74         self.assertEqual(
  75             self.helm_conn._local_async_exec.call_count,
  76             2,
  77             "local_async_exec expected 2 calls, called {}".format(
  78                 self.helm_conn._local_async_exec.call_count
  79             ),
  80         )
  81
  82         repo_update_command = (
  83             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo update {}"
  84         ).format(repo_name)
  85         repo_add_command = (
  86             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo add {} {}"
  87         ).format(repo_name, repo_url)
  88         calls = self.helm_conn._local_async_exec.call_args_list
  89         call0_kargs = calls[0][1]
  90         self.assertEqual(
  91             call0_kargs.get("command"),
  92             repo_add_command,
  93             "Invalid repo add command: {}".format(call0_kargs.get("command")),
  94         )
  95         self.assertEqual(
  96             call0_kargs.get("env"),
  97             self.env,
  98             "Invalid env for add command: {}".format(call0_kargs.get("env")),
  99         )
 100         call1_kargs = calls[1][1]
 101         self.assertEqual(
 102             call1_kargs.get("command"),
 103             repo_update_command,
 104             "Invalid repo update command: {}".format(call1_kargs.get("command")),
 105         )
 106         self.assertEqual(
 107             call1_kargs.get("env"),
 108             self.env,
 109             "Invalid env for update command: {}".format(call1_kargs.get("env")),
 110         )
 111
 112     @asynctest.fail_on(active_handles=True)
 113     async def test_repo_list(self):
 114         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 115
 116         await self.helm_conn.repo_list(self.cluster_uuid)
 117
 118         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 119         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 120             from_path=self.cluster_id
 121         )
 122         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo list --output yaml"
 123         self.helm_conn._local_async_exec.assert_called_with(
 124             command=command, env=self.env, raise_exception_on_error=False
 125         )
 126
 127     @asynctest.fail_on(active_handles=True)
 128     async def test_repo_remove(self):
 129         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 130         repo_name = "bitnami"
 131         await self.helm_conn.repo_remove(self.cluster_uuid, repo_name)
 132
 133         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 134         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 135             from_path=self.cluster_id
 136         )
 137         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm repo remove {}".format(
 138             repo_name
 139         )
 140         self.helm_conn._local_async_exec.assert_called_once_with(
 141             command=command, env=self.env, raise_exception_on_error=True
 142         )
 143
 144     @asynctest.fail_on(active_handles=True)
 145     async def test_install(self):
 146         kdu_model = "stable/openldap:1.2.2"
 147         kdu_instance = "stable-openldap-0005399828"
 148         db_dict = {}
 149         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 150         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=None)
 151         self.helm_conn._store_status = asynctest.CoroutineMock()
 152         self.helm_conn.generate_kdu_instance_name = Mock(return_value=kdu_instance)
 153
 154         await self.helm_conn.install(
 155             self.cluster_uuid,
 156             kdu_model,
 157             kdu_instance,
 158             atomic=True,
 159             namespace=self.namespace,
 160             db_dict=db_dict,
 161         )
 162
 163         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 164         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 165             from_path=self.cluster_id
 166         )
 167         self.helm_conn._store_status.assert_called_with(
 168             cluster_id=self.cluster_id,
 169             kdu_instance=kdu_instance,
 170             namespace=self.namespace,
 171             db_dict=db_dict,
 172             operation="install",
 173             run_once=True,
 174             check_every=0,
 175         )
 176         command = (
 177             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm install "
 178             "--atomic --output yaml   --timeout 300 "
 179             "--name=stable-openldap-0005399828 --namespace testk8s stable/openldap "
 180             "--version 1.2.2"
 181         )
 182         self.helm_conn._local_async_exec.assert_called_once_with(
 183             command=command, env=self.env, raise_exception_on_error=False
 184         )
 185
 186     @asynctest.fail_on(active_handles=True)
 187     async def test_upgrade(self):
 188         kdu_model = "stable/openldap:1.2.3"
 189         kdu_instance = "stable-openldap-0005399828"
 190         db_dict = {}
 191         instance_info = {
 192             "chart": "openldap-1.2.2",
 193             "name": kdu_instance,
 194             "namespace": self.namespace,
 195             "revision": 1,
 196             "status": "DEPLOYED",
 197         }
 198         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 199         self.helm_conn._store_status = asynctest.CoroutineMock()
 200         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 201             return_value=instance_info
 202         )
 203
 204         await self.helm_conn.upgrade(
 205             self.cluster_uuid, kdu_instance, kdu_model, atomic=True, db_dict=db_dict
 206         )
 207         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 208         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 209             from_path=self.cluster_id
 210         )
 211         self.helm_conn._store_status.assert_called_with(
 212             cluster_id=self.cluster_id,
 213             kdu_instance=kdu_instance,
 214             namespace=self.namespace,
 215             db_dict=db_dict,
 216             operation="upgrade",
 217             run_once=True,
 218             check_every=0,
 219         )
 220         command = (
 221             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm upgrade "
 222             "--atomic --output yaml  --timeout 300 stable-openldap-0005399828 stable/openldap --version 1.2.3"
 223         )
 224         self.helm_conn._local_async_exec.assert_called_once_with(
 225             command=command, env=self.env, raise_exception_on_error=False
 226         )
 227
 228     @asynctest.fail_on(active_handles=True)
 229     async def test_scale(self):
 230         kdu_model = "stable/openldap:1.2.3"
 231         kdu_instance = "stable-openldap-0005399828"
 232         db_dict = {}
 233         instance_info = {
 234             "chart": "openldap-1.2.3",
 235             "name": kdu_instance,
 236             "namespace": self.namespace,
 237             "revision": 1,
 238             "status": "DEPLOYED",
 239         }
 240         repo_list = [
 241             {
 242                 "name": "stable",
 243                 "url": "https://kubernetes-charts.storage.googleapis.com/",
 244             }
 245         ]
 246         kdu_values = """
 247             # Default values for openldap.
 248             # This is a YAML-formatted file.
 249             # Declare variables to be passed into your templates.
 250
 251             replicaCount: 1
 252             dummy-app:
 253               replicas: 2
 254         """
 255
 256         self.helm_conn.repo_list = asynctest.CoroutineMock(return_value=repo_list)
 257         self.helm_conn.values_kdu = asynctest.CoroutineMock(return_value=kdu_values)
 258         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 259         self.helm_conn._store_status = asynctest.CoroutineMock()
 260         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 261             return_value=instance_info
 262         )
 263
 264         # TEST-1
 265         await self.helm_conn.scale(
 266             kdu_instance,
 267             2,
 268             "",
 269             kdu_model=kdu_model,
 270             cluster_uuid=self.cluster_uuid,
 271             atomic=True,
 272             db_dict=db_dict,
 273         )
 274         command = (
 275             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 276             "/usr/bin/helm upgrade --atomic --output yaml --set replicaCount=2 "
 277             "--timeout 1800s stable-openldap-0005399828 stable/openldap "
 278             "--version 1.2.3"
 279         )
 280         self.helm_conn._local_async_exec.assert_called_once_with(
 281             command=command, env=self.env, raise_exception_on_error=False
 282         )
 283
 284         # TEST-2
 285         await self.helm_conn.scale(
 286             kdu_instance,
 287             3,
 288             "dummy-app",
 289             kdu_model=kdu_model,
 290             cluster_uuid=self.cluster_uuid,
 291             atomic=True,
 292             db_dict=db_dict,
 293         )
 294         command = (
 295             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 296             "/usr/bin/helm upgrade --atomic --output yaml --set dummy-app.replicas=3 "
 297             "--timeout 1800s stable-openldap-0005399828 stable/openldap "
 298             "--version 1.2.3"
 299         )
 300         self.helm_conn._local_async_exec.assert_called_with(
 301             command=command, env=self.env, raise_exception_on_error=False
 302         )
 303         self.helm_conn.fs.reverse_sync.assert_called_with(from_path=self.cluster_id)
 304         self.helm_conn._store_status.assert_called_with(
 305             cluster_id=self.cluster_id,
 306             kdu_instance=kdu_instance,
 307             namespace=self.namespace,
 308             db_dict=db_dict,
 309             operation="scale",
 310             run_once=True,
 311             check_every=0,
 312         )
 313
 314     @asynctest.fail_on(active_handles=True)
 315     async def test_rollback(self):
 316         kdu_instance = "stable-openldap-0005399828"
 317         db_dict = {}
 318         instance_info = {
 319             "chart": "openldap-1.2.3",
 320             "name": kdu_instance,
 321             "namespace": self.namespace,
 322             "revision": 2,
 323             "status": "DEPLOYED",
 324         }
 325         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 326         self.helm_conn._store_status = asynctest.CoroutineMock()
 327         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 328             return_value=instance_info
 329         )
 330
 331         await self.helm_conn.rollback(
 332             self.cluster_uuid, kdu_instance=kdu_instance, revision=1, db_dict=db_dict
 333         )
 334         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 335         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 336             from_path=self.cluster_id
 337         )
 338         self.helm_conn._store_status.assert_called_with(
 339             cluster_id=self.cluster_id,
 340             kdu_instance=kdu_instance,
 341             namespace=self.namespace,
 342             db_dict=db_dict,
 343             operation="rollback",
 344             run_once=True,
 345             check_every=0,
 346         )
 347         command = (
 348             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config "
 349             "/usr/bin/helm rollback stable-openldap-0005399828 1 --wait"
 350         )
 351         self.helm_conn._local_async_exec.assert_called_once_with(
 352             command=command, env=self.env, raise_exception_on_error=False
 353         )
 354
 355     @asynctest.fail_on(active_handles=True)
 356     async def test_uninstall(self):
 357         kdu_instance = "stable-openldap-0005399828"
 358         instance_info = {
 359             "chart": "openldap-1.2.2",
 360             "name": kdu_instance,
 361             "namespace": self.namespace,
 362             "revision": 3,
 363             "status": "DEPLOYED",
 364         }
 365         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 366         self.helm_conn._store_status = asynctest.CoroutineMock()
 367         self.helm_conn.get_instance_info = asynctest.CoroutineMock(
 368             return_value=instance_info
 369         )
 370
 371         await self.helm_conn.uninstall(self.cluster_uuid, kdu_instance)
 372         self.helm_conn.fs.sync.assert_called_with(from_path=self.cluster_id)
 373         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 374             from_path=self.cluster_id
 375         )
 376         command = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm delete --purge  {}".format(
 377             kdu_instance
 378         )
 379         self.helm_conn._local_async_exec.assert_called_once_with(
 380             command=command, env=self.env, raise_exception_on_error=True
 381         )
 382
 383     @asynctest.fail_on(active_handles=True)
 384     async def test_get_services(self):
 385         kdu_instance = "test_services_1"
 386         service = {"name": "testservice", "type": "LoadBalancer"}
 387         self.helm_conn._local_async_exec_pipe = asynctest.CoroutineMock(
 388             return_value=("", 0)
 389         )
 390         self.helm_conn._parse_services = Mock(return_value=["testservice"])
 391         self.helm_conn._get_service = asynctest.CoroutineMock(return_value=service)
 392
 393         services = await self.helm_conn.get_services(
 394             self.cluster_uuid, kdu_instance, self.namespace
 395         )
 396         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 397         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 398             from_path=self.cluster_id
 399         )
 400         self.helm_conn._parse_services.assert_called_once()
 401         command1 = "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm get manifest {} ".format(
 402             kdu_instance
 403         )
 404         command2 = "/usr/bin/kubectl get --namespace={} -f -".format(self.namespace)
 405         self.helm_conn._local_async_exec_pipe.assert_called_once_with(
 406             command1, command2, env=self.env, raise_exception_on_error=True
 407         )
 408         self.assertEqual(
 409             services, [service], "Invalid service returned from get_service"
 410         )
 411
 412     @asynctest.fail_on(active_handles=True)
 413     async def test_get_service(self):
 414         service_name = "service1"
 415
 416         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 417         await self.helm_conn.get_service(
 418             self.cluster_uuid, service_name, self.namespace
 419         )
 420
 421         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 422         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 423             from_path=self.cluster_id
 424         )
 425         command = (
 426             "/usr/bin/kubectl --kubeconfig=./tmp/helm_cluster_id/.kube/config "
 427             "--namespace=testk8s get service service1 -o=yaml"
 428         )
 429         self.helm_conn._local_async_exec.assert_called_once_with(
 430             command=command, env=self.env, raise_exception_on_error=True
 431         )
 432
 433     @asynctest.fail_on(active_handles=True)
 434     async def test_inspect_kdu(self):
 435         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 436
 437         kdu_model = "stable/openldap:1.2.4"
 438         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 439         await self.helm_conn.inspect_kdu(kdu_model, repo_url)
 440
 441         command = (
 442             "/usr/bin/helm inspect  openldap --repo "
 443             "https://kubernetes-charts.storage.googleapis.com/ "
 444             "--version 1.2.4"
 445         )
 446         self.helm_conn._local_async_exec.assert_called_with(command=command)
 447
 448     @asynctest.fail_on(active_handles=True)
 449     async def test_help_kdu(self):
 450         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 451
 452         kdu_model = "stable/openldap:1.2.4"
 453         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 454         await self.helm_conn.help_kdu(kdu_model, repo_url)
 455
 456         command = (
 457             "/usr/bin/helm inspect readme openldap --repo "
 458             "https://kubernetes-charts.storage.googleapis.com/ "
 459             "--version 1.2.4"
 460         )
 461         self.helm_conn._local_async_exec.assert_called_with(command=command)
 462
 463     @asynctest.fail_on(active_handles=True)
 464     async def test_values_kdu(self):
 465         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 466
 467         kdu_model = "stable/openldap:1.2.4"
 468         repo_url = "https://kubernetes-charts.storage.googleapis.com/"
 469         await self.helm_conn.values_kdu(kdu_model, repo_url)
 470
 471         command = (
 472             "/usr/bin/helm inspect values openldap --repo "
 473             "https://kubernetes-charts.storage.googleapis.com/ "
 474             "--version 1.2.4"
 475         )
 476         self.helm_conn._local_async_exec.assert_called_with(command=command)
 477
 478     @asynctest.fail_on(active_handles=True)
 479     async def test_get_values_kdu(self):
 480         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 481
 482         kdu_instance = "stable-openldap-0005399828"
 483         await self.helm_conn.get_values_kdu(
 484             kdu_instance, self.namespace, self.env["KUBECONFIG"]
 485         )
 486
 487         command = (
 488             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm get values "
 489             "stable-openldap-0005399828 --output yaml"
 490         )
 491         self.helm_conn._local_async_exec.assert_called_with(command=command)
 492
 493     @asynctest.fail_on(active_handles=True)
 494     async def test_instances_list(self):
 495         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 496
 497         await self.helm_conn.instances_list(self.cluster_uuid)
 498         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 499         self.helm_conn.fs.reverse_sync.assert_called_once_with(
 500             from_path=self.cluster_id
 501         )
 502         command = "/usr/bin/helm list --output yaml"
 503         self.helm_conn._local_async_exec.assert_called_once_with(
 504             command=command, env=self.env, raise_exception_on_error=True
 505         )
 506
 507     @asynctest.fail_on(active_handles=True)
 508     async def test_status_kdu(self):
 509         kdu_instance = "stable-openldap-0005399828"
 510         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 511
 512         await self.helm_conn._status_kdu(
 513             self.cluster_id, kdu_instance, self.namespace, yaml_format=True
 514         )
 515         command = (
 516             "env KUBECONFIG=./tmp/helm_cluster_id/.kube/config /usr/bin/helm status {} --output yaml"
 517         ).format(kdu_instance)
 518         self.helm_conn._local_async_exec.assert_called_once_with(
 519             command=command,
 520             env=self.env,
 521             raise_exception_on_error=True,
 522             show_error_log=False,
 523         )
 524
 525     @asynctest.fail_on(active_handles=True)
 526     async def test_store_status(self):
 527         kdu_instance = "stable-openldap-0005399828"
 528         db_dict = {}
 529         status = {
 530             "info": {
 531                 "description": "Install complete",
 532                 "status": {
 533                     "code": "1",
 534                     "notes": "The openldap helm chart has been installed",
 535                 },
 536             }
 537         }
 538         self.helm_conn._status_kdu = asynctest.CoroutineMock(return_value=status)
 539         self.helm_conn.write_app_status_to_db = asynctest.CoroutineMock(
 540             return_value=status
 541         )
 542
 543         await self.helm_conn._store_status(
 544             cluster_id=self.cluster_id,
 545             kdu_instance=kdu_instance,
 546             namespace=self.namespace,
 547             db_dict=db_dict,
 548             operation="install",
 549             run_once=True,
 550             check_every=0,
 551         )
 552         self.helm_conn._status_kdu.assert_called_once_with(
 553             cluster_id=self.cluster_id,
 554             kdu_instance=kdu_instance,
 555             namespace=self.namespace,
 556             yaml_format=False,
 557         )
 558         self.helm_conn.write_app_status_to_db.assert_called_once_with(
 559             db_dict=db_dict,
 560             status="Install complete",
 561             detailed_status=str(status),
 562             operation="install",
 563         )
 564
 565     @asynctest.fail_on(active_handles=True)
 566     async def test_reset_uninstall_false(self):
 567         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 568
 569         await self.helm_conn.reset(self.cluster_uuid, force=False, uninstall_sw=False)
 570         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 571         self.helm_conn.fs.file_delete.assert_called_once_with(
 572             self.cluster_id, ignore_non_exist=True
 573         )
 574         self.helm_conn._uninstall_sw.assert_not_called()
 575
 576     @asynctest.fail_on(active_handles=True)
 577     async def test_reset_uninstall(self):
 578         kdu_instance = "stable-openldap-0021099429"
 579         instances = [
 580             {
 581                 "app_version": "2.4.48",
 582                 "chart": "openldap-1.2.3",
 583                 "name": kdu_instance,
 584                 "namespace": self.namespace,
 585                 "revision": "1",
 586                 "status": "deployed",
 587                 "updated": "2020-10-30 11:11:20.376744191 +0000 UTC",
 588             }
 589         ]
 590         self.helm_conn._get_namespace = Mock(return_value=self.namespace)
 591         self.helm_conn._uninstall_sw = asynctest.CoroutineMock()
 592         self.helm_conn.instances_list = asynctest.CoroutineMock(return_value=instances)
 593         self.helm_conn.uninstall = asynctest.CoroutineMock()
 594
 595         await self.helm_conn.reset(self.cluster_uuid, force=True, uninstall_sw=True)
 596         self.helm_conn.fs.sync.assert_called_once_with(from_path=self.cluster_id)
 597         self.helm_conn.fs.file_delete.assert_called_once_with(
 598             self.cluster_id, ignore_non_exist=True
 599         )
 600         self.helm_conn._get_namespace.assert_called_once_with(
 601             cluster_uuid=self.cluster_uuid
 602         )
 603         self.helm_conn.instances_list.assert_called_once_with(
 604             cluster_uuid=self.cluster_uuid
 605         )
 606         self.helm_conn.uninstall.assert_called_once_with(
 607             cluster_uuid=self.cluster_uuid, kdu_instance=kdu_instance
 608         )
 609         self.helm_conn._uninstall_sw.assert_called_once_with(
 610             cluster_id=self.cluster_id, namespace=self.namespace
 611         )
 612
 613     @asynctest.fail_on(active_handles=True)
 614     async def test_uninstall_sw_namespace(self):
 615         self.helm_conn._local_async_exec = asynctest.CoroutineMock(return_value=("", 0))
 616
 617         await self.helm_conn._uninstall_sw(self.cluster_id, self.namespace)
 618         calls = self.helm_conn._local_async_exec.call_args_list
 619         self.assertEqual(
 620             len(calls), 3, "To uninstall should have executed three commands"
 621         )
 622         call0_kargs = calls[0][1]
 623         command_0 = "/usr/bin/helm --kubeconfig={} --home={} reset".format(
 624             self.kube_config, self.helm_home
 625         )
 626         self.assertEqual(
 627             call0_kargs,
 628             {"command": command_0, "raise_exception_on_error": True, "env": self.env},
 629             "Invalid args for first call to local_exec",
 630         )
 631         call1_kargs = calls[1][1]
 632         command_1 = (
 633             "/usr/bin/kubectl --kubeconfig={} delete "
 634             "clusterrolebinding.rbac.authorization.k8s.io/osm-tiller-cluster-rule".format(
 635                 self.kube_config
 636             )
 637         )
 638         self.assertEqual(
 639             call1_kargs,
 640             {"command": command_1, "raise_exception_on_error": False, "env": self.env},
 641             "Invalid args for second call to local_exec",
 642         )
 643         call2_kargs = calls[2][1]
 644         command_2 = (
 645             "/usr/bin/kubectl --kubeconfig={} --namespace {} delete "
 646             "serviceaccount/{}".format(
 647                 self.kube_config, self.namespace, self.service_account
 648             )
 649         )
 650         self.assertEqual(
 651             call2_kargs,
 652             {"command": command_2, "raise_exception_on_error": False, "env": self.env},
 653             "Invalid args for third call to local_exec",
 654         )