X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=n2vc%2Fk8s_helm_base_conn.py;h=2a2e9354f9728fce721d330877828d9a91d1d3a3;hb=6d6707771792dd1a917b32c77b40920760aa0a08;hp=b3d7bda3c83f40812ccf03d2a196ea247099dc48;hpb=82b591ceed704c798ead2d9104085a08e75b511b;p=osm%2FN2VC.git

diff --git a/n2vc/k8s_helm_base_conn.py b/n2vc/k8s_helm_base_conn.py
index b3d7bda..2a2e935 100644
--- a/n2vc/k8s_helm_base_conn.py
+++ b/n2vc/k8s_helm_base_conn.py
@@ -21,16 +21,17 @@
 ##
 import abc
 import asyncio
+from typing import Union
 import random
 import time
 import shlex
 import shutil
 import stat
-import subprocess
 import os
 import yaml
 from uuid import uuid4
 
+from n2vc.config import EnvironConfig
 from n2vc.exceptions import K8sException
 from n2vc.k8s_conn import K8sConnector
 
@@ -44,7 +45,6 @@ class K8sHelmBaseConnector(K8sConnector):
     """
 
     service_account = "osm"
-    _STABLE_REPO_URL = "https://charts.helm.sh/stable"
 
     def __init__(
         self,
@@ -54,7 +54,6 @@ class K8sHelmBaseConnector(K8sConnector):
         helm_command: str = "/usr/bin/helm",
         log: object = None,
         on_update_db=None,
-        vca_config: dict = None,
     ):
         """
 
@@ -71,6 +70,7 @@ class K8sHelmBaseConnector(K8sConnector):
 
         self.log.info("Initializing K8S Helm connector")
 
+        self.config = EnvironConfig()
         # random numbers for release name generation
         random.seed(time.time())
 
@@ -86,10 +86,9 @@ class K8sHelmBaseConnector(K8sConnector):
         self._check_file_exists(filename=helm_command, exception_if_not_exists=True)
 
         # obtain stable repo url from config or apply default
-        if not vca_config or not vca_config.get("stablerepourl"):
-            self._stable_repo_url = self._STABLE_REPO_URL
-        else:
-            self._stable_repo_url = vca_config.get("stablerepourl")
+        self._stable_repo_url = self.config.get("stablerepourl")
+        if self._stable_repo_url == "None":
+            self._stable_repo_url = None
 
     @staticmethod
     def _get_namespace_cluster_id(cluster_uuid: str) -> (str, str):
@@ -160,26 +159,53 @@ class K8sHelmBaseConnector(K8sConnector):
             )
         )
 
-        # sync local dir
-        self.fs.sync(from_path=cluster_id)
-
         # init_env
         paths, env = self._init_paths_env(
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
+        # helm repo add name url
+        command = "env KUBECONFIG={} {} repo add {} {}".format(
+            paths["kube_config"], self._helm_command, name, url
+        )
+        self.log.debug("adding repo: {}".format(command))
+        await self._local_async_exec(
+            command=command, raise_exception_on_error=True, env=env
+        )
+
         # helm repo update
-        command = "{} repo update".format(self._helm_command)
+        command = "env KUBECONFIG={} {} repo update {}".format(
+            paths["kube_config"], self._helm_command, name
+        )
         self.log.debug("updating repo: {}".format(command))
         await self._local_async_exec(
             command=command, raise_exception_on_error=False, env=env
         )
 
-        # helm repo add name url
-        command = "{} repo add {} {}".format(self._helm_command, name, url)
-        self.log.debug("adding repo: {}".format(command))
+        # sync fs
+        self.fs.reverse_sync(from_path=cluster_id)
+
+    async def repo_update(self, cluster_id: str, name: str, repo_type: str = "chart"):
+        self.log.debug(
+            "Cluster {}, updating {} repository {}".format(cluster_id, repo_type, name)
+        )
+
+        # init_env
+        paths, env = self._init_paths_env(
+            cluster_name=cluster_id, create_if_not_exist=True
+        )
+
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
+        # helm repo update
+        command = "{} repo update {}".format(self._helm_command, name)
+        self.log.debug("updating repo: {}".format(command))
         await self._local_async_exec(
-            command=command, raise_exception_on_error=True, env=env
+            command=command, raise_exception_on_error=False, env=env
         )
 
         # sync fs
@@ -195,15 +221,17 @@ class K8sHelmBaseConnector(K8sConnector):
         _, cluster_id = self._get_namespace_cluster_id(cluster_uuid)
         self.log.debug("list repositories for cluster {}".format(cluster_id))
 
-        # sync local dir
-        self.fs.sync(from_path=cluster_id)
-
         # config filename
         paths, env = self._init_paths_env(
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
-        command = "{} repo list --output yaml".format(self._helm_command)
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
+        command = "env KUBECONFIG={} {} repo list --output yaml".format(
+            paths["kube_config"], self._helm_command
+        )
 
         # Set exception to false because if there are no repos just want an empty list
         output, _rc = await self._local_async_exec(
@@ -228,15 +256,17 @@ class K8sHelmBaseConnector(K8sConnector):
         _, cluster_id = self._get_namespace_cluster_id(cluster_uuid)
         self.log.debug("remove {} repositories for cluster {}".format(name, cluster_id))
 
-        # sync local dir
-        self.fs.sync(from_path=cluster_id)
-
         # init env, paths
         paths, env = self._init_paths_env(
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
-        command = "{} repo remove {}".format(self._helm_command, name)
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
+        command = "env KUBECONFIG={} {} repo remove {}".format(
+            paths["kube_config"], self._helm_command, name
+        )
         await self._local_async_exec(
             command=command, raise_exception_on_error=True, env=env
         )
@@ -330,6 +360,11 @@ class K8sHelmBaseConnector(K8sConnector):
         kdu_name: str = None,
         namespace: str = None,
     ):
+        # init env, paths
+        paths, env = self._init_paths_env(
+            cluster_name=cluster_id, create_if_not_exist=True
+        )
+
         # params to str
         params_str, file_to_delete = self._params_to_file_option(
             cluster_id=cluster_id, params=params
@@ -343,8 +378,19 @@ class K8sHelmBaseConnector(K8sConnector):
                 version = str(parts[1])
                 kdu_model = parts[0]
 
+        repo = self._split_repo(kdu_model)
+        if repo:
+            await self.repo_update(cluster_id, repo)
+
         command = self._get_install_command(
-            kdu_model, kdu_instance, namespace, params_str, version, atomic, timeout
+            kdu_model,
+            kdu_instance,
+            namespace,
+            params_str,
+            version,
+            atomic,
+            timeout,
+            paths["kube_config"],
         )
 
         self.log.debug("installing: {}".format(command))
@@ -365,7 +411,6 @@ class K8sHelmBaseConnector(K8sConnector):
                     namespace=namespace,
                     db_dict=db_dict,
                     operation="install",
-                    run_once=False,
                 )
             )
 
@@ -394,8 +439,6 @@ class K8sHelmBaseConnector(K8sConnector):
             namespace=namespace,
             db_dict=db_dict,
             operation="install",
-            run_once=True,
-            check_every=0,
         )
 
         if rc != 0:
@@ -429,6 +472,9 @@ class K8sHelmBaseConnector(K8sConnector):
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
         # params to str
         params_str, file_to_delete = self._params_to_file_option(
             cluster_id=cluster_id, params=params
@@ -442,6 +488,10 @@ class K8sHelmBaseConnector(K8sConnector):
                 version = str(parts[1])
                 kdu_model = parts[0]
 
+        repo = self._split_repo(kdu_model)
+        if repo:
+            await self.repo_update(cluster_id, repo)
+
         command = self._get_upgrade_command(
             kdu_model,
             kdu_instance,
@@ -450,6 +500,7 @@ class K8sHelmBaseConnector(K8sConnector):
             version,
             atomic,
             timeout,
+            paths["kube_config"],
         )
 
         self.log.debug("upgrading: {}".format(command))
@@ -470,7 +521,6 @@ class K8sHelmBaseConnector(K8sConnector):
                     namespace=instance_info["namespace"],
                     db_dict=db_dict,
                     operation="upgrade",
-                    run_once=False,
                 )
             )
 
@@ -498,8 +548,6 @@ class K8sHelmBaseConnector(K8sConnector):
             namespace=instance_info["namespace"],
             db_dict=db_dict,
             operation="upgrade",
-            run_once=True,
-            check_every=0,
         )
 
         if rc != 0:
@@ -563,8 +611,11 @@ class K8sHelmBaseConnector(K8sConnector):
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
         command = self._get_rollback_command(
-            kdu_instance, instance_info["namespace"], revision
+            kdu_instance, instance_info["namespace"], revision, paths["kube_config"]
         )
 
         self.log.debug("rolling_back: {}".format(command))
@@ -583,7 +634,6 @@ class K8sHelmBaseConnector(K8sConnector):
                 namespace=instance_info["namespace"],
                 db_dict=db_dict,
                 operation="rollback",
-                run_once=False,
             )
         )
 
@@ -602,8 +652,6 @@ class K8sHelmBaseConnector(K8sConnector):
             namespace=instance_info["namespace"],
             db_dict=db_dict,
             operation="rollback",
-            run_once=True,
-            check_every=0,
         )
 
         if rc != 0:
@@ -648,14 +696,19 @@ class K8sHelmBaseConnector(K8sConnector):
         # look for instance to obtain namespace
         instance_info = await self.get_instance_info(cluster_uuid, kdu_instance)
         if not instance_info:
-            raise K8sException("kdu_instance {} not found".format(kdu_instance))
-
+            self.log.warning(("kdu_instance {} not found".format(kdu_instance)))
+            return True
         # init env, paths
         paths, env = self._init_paths_env(
             cluster_name=cluster_id, create_if_not_exist=True
         )
 
-        command = self._get_uninstall_command(kdu_instance, instance_info["namespace"])
+        # sync local dir
+        self.fs.sync(from_path=cluster_id)
+
+        command = self._get_uninstall_command(
+            kdu_instance, instance_info["namespace"], paths["kube_config"]
+        )
         output, _rc = await self._local_async_exec(
             command=command, raise_exception_on_error=True, env=env
         )
@@ -748,11 +801,18 @@ class K8sHelmBaseConnector(K8sConnector):
             )
         )
 
+        # init env, paths
+        paths, env = self._init_paths_env(
+            cluster_name=cluster_id, create_if_not_exist=True
+        )
+
         # sync local dir
         self.fs.sync(from_path=cluster_id)
 
         # get list of services names for kdu
-        service_names = await self._get_services(cluster_id, kdu_instance, namespace)
+        service_names = await self._get_services(
+            cluster_id, kdu_instance, namespace, paths["kube_config"]
+        )
 
         service_list = []
         for service in service_names:
@@ -786,7 +846,9 @@ class K8sHelmBaseConnector(K8sConnector):
 
         return service
 
-    async def status_kdu(self, cluster_uuid: str, kdu_instance: str, **kwargs) -> str:
+    async def status_kdu(
+        self, cluster_uuid: str, kdu_instance: str, yaml_format: str = False, **kwargs
+    ) -> Union[str, dict]:
         """
         This call would retrieve tha current state of a given KDU instance. It would be
         would allow to retrieve the _composition_ (i.e. K8s objects) and _specific
@@ -796,6 +858,8 @@ class K8sHelmBaseConnector(K8sConnector):
         :param cluster_uuid: UUID of a K8s cluster known by OSM
         :param kdu_instance: unique name for the KDU instance
         :param kwargs: Additional parameters (None yet)
+        :param yaml_format: if the return shall be returned as an YAML string or as a
+                                dictionary
         :return: If successful, it will return the following vector of arguments:
         - K8s `namespace` in the cluster where the KDU lives
         - `state` of the KDU instance. It can be:
@@ -838,8 +902,8 @@ class K8sHelmBaseConnector(K8sConnector):
             cluster_id=cluster_id,
             kdu_instance=kdu_instance,
             namespace=instance["namespace"],
+            yaml_format=yaml_format,
             show_error_log=True,
-            return_text=True,
         )
 
         # sync fs
@@ -972,7 +1036,7 @@ class K8sHelmBaseConnector(K8sConnector):
         """
 
     @abc.abstractmethod
-    async def _get_services(self, cluster_id, kdu_instance, namespace):
+    async def _get_services(self, cluster_id, kdu_instance, namespace, kubeconfig):
         """
         Implements the helm version dependent method to obtain services from a helm instance
         """
@@ -983,16 +1047,24 @@ class K8sHelmBaseConnector(K8sConnector):
         cluster_id: str,
         kdu_instance: str,
         namespace: str = None,
+        yaml_format: bool = False,
         show_error_log: bool = False,
-        return_text: bool = False,
-    ):
+    ) -> Union[str, dict]:
         """
         Implements the helm version dependent method to obtain status of a helm instance
         """
 
     @abc.abstractmethod
     def _get_install_command(
-        self, kdu_model, kdu_instance, namespace, params_str, version, atomic, timeout
+        self,
+        kdu_model,
+        kdu_instance,
+        namespace,
+        params_str,
+        version,
+        atomic,
+        timeout,
+        kubeconfig,
     ) -> str:
         """
         Obtain command to be executed to delete the indicated instance
@@ -1000,20 +1072,32 @@ class K8sHelmBaseConnector(K8sConnector):
 
     @abc.abstractmethod
     def _get_upgrade_command(
-        self, kdu_model, kdu_instance, namespace, params_str, version, atomic, timeout
+        self,
+        kdu_model,
+        kdu_instance,
+        namespace,
+        params_str,
+        version,
+        atomic,
+        timeout,
+        kubeconfig,
     ) -> str:
         """
         Obtain command to be executed to upgrade the indicated instance
         """
 
     @abc.abstractmethod
-    def _get_rollback_command(self, kdu_instance, namespace, revision) -> str:
+    def _get_rollback_command(
+        self, kdu_instance, namespace, revision, kubeconfig
+    ) -> str:
         """
         Obtain command to be executed to rollback the indicated instance
         """
 
     @abc.abstractmethod
-    def _get_uninstall_command(self, kdu_instance: str, namespace: str) -> str:
+    def _get_uninstall_command(
+        self, kdu_instance: str, namespace: str, kubeconfig: str
+    ) -> str:
         """
         Obtain command to be executed to delete the indicated instance
         """
@@ -1137,27 +1221,12 @@ class K8sHelmBaseConnector(K8sConnector):
         of dictionaries
         """
         new_list = []
-        for dictionary in input_list:
-            new_dict = dict((k.lower(), v) for k, v in dictionary.items())
-            new_list.append(new_dict)
+        if input_list:
+            for dictionary in input_list:
+                new_dict = dict((k.lower(), v) for k, v in dictionary.items())
+                new_list.append(new_dict)
         return new_list
 
-    def _local_exec(self, command: str) -> (str, int):
-        command = self._remove_multiple_spaces(command)
-        self.log.debug("Executing sync local command: {}".format(command))
-        # raise exception if fails
-        output = ""
-        try:
-            output = subprocess.check_output(
-                command, shell=True, universal_newlines=True
-            )
-            return_code = 0
-            self.log.debug(output)
-        except Exception:
-            return_code = 1
-
-        return output, return_code
-
     async def _local_async_exec(
         self,
         command: str,
@@ -1382,42 +1451,51 @@ class K8sHelmBaseConnector(K8sConnector):
         operation: str,
         kdu_instance: str,
         namespace: str = None,
-        check_every: float = 10,
         db_dict: dict = None,
-        run_once: bool = False,
-    ):
-        while True:
-            try:
-                await asyncio.sleep(check_every)
-                detailed_status = await self._status_kdu(
-                    cluster_id=cluster_id,
-                    kdu_instance=kdu_instance,
-                    namespace=namespace,
-                    return_text=False,
-                )
-                status = detailed_status.get("info").get("description")
-                self.log.debug("KDU {} STATUS: {}.".format(kdu_instance, status))
-                # write status to db
-                result = await self.write_app_status_to_db(
-                    db_dict=db_dict,
-                    status=str(status),
-                    detailed_status=str(detailed_status),
-                    operation=operation,
-                )
-                if not result:
-                    self.log.info("Error writing in database. Task exiting...")
-                    return
-            except asyncio.CancelledError:
-                self.log.debug("Task cancelled")
-                return
-            except Exception as e:
-                self.log.debug(
-                    "_store_status exception: {}".format(str(e)), exc_info=True
-                )
-                pass
-            finally:
-                if run_once:
-                    return
+    ) -> None:
+        """
+        Obtains the status of the KDU instance based on Helm Charts, and stores it in the database.
+
+        :param cluster_id (str): the cluster where the KDU instance is deployed
+        :param operation (str): The operation related to the status to be updated (for instance, "install" or "upgrade")
+        :param kdu_instance (str): The KDU instance in relation to which the status is obtained
+        :param namespace (str): The Kubernetes namespace where the KDU instance was deployed. Defaults to None
+        :param db_dict (dict): A dictionary with the database necessary information. It shall contain the
+        values for the keys:
+            - "collection": The Mongo DB collection to write to
+            - "filter": The query filter to use in the update process
+            - "path": The dot separated keys which targets the object to be updated
+        Defaults to None.
+        """
+
+        try:
+            detailed_status = await self._status_kdu(
+                cluster_id=cluster_id,
+                kdu_instance=kdu_instance,
+                yaml_format=False,
+                namespace=namespace,
+            )
+
+            status = detailed_status.get("info").get("description")
+            self.log.debug(f"Status for KDU {kdu_instance} obtained: {status}.")
+
+            # write status to db
+            result = await self.write_app_status_to_db(
+                db_dict=db_dict,
+                status=str(status),
+                detailed_status=str(detailed_status),
+                operation=operation,
+            )
+
+            if not result:
+                self.log.info("Error writing in database. Task exiting...")
+
+        except asyncio.CancelledError as e:
+            self.log.warning(
+                f"Exception in method {self._store_status.__name__} (task cancelled): {e}"
+            )
+        except Exception as e:
+            self.log.warning(f"Exception in method {self._store_status.__name__}: {e}")
 
     # params for use in -f file
     # returns values file option and filename (in order to delete it at the end)
@@ -1500,3 +1578,10 @@ class K8sHelmBaseConnector(K8sConnector):
 
         name = name + get_random_number()
         return name.lower()
+
+    def _split_repo(self, kdu_model: str) -> str:
+        repo_name = None
+        idx = kdu_model.find("/")
+        if idx >= 0:
+            repo_name = kdu_model[:idx]
+        return repo_name