Refactor cluster checks to use common_check_list and readiness_loop
Change-Id: I3a1b28b7139c863805b5d66a461dc4a3a6baded8
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
diff --git a/osm_lcm/odu_libs/cluster_mgmt.py b/osm_lcm/odu_libs/cluster_mgmt.py
index a86ce20..1c21675 100644
--- a/osm_lcm/odu_libs/cluster_mgmt.py
+++ b/osm_lcm/odu_libs/cluster_mgmt.py
@@ -17,11 +17,8 @@
from pyrage import x25519
-import asyncio
-import traceback
import yaml
import base64
-from math import ceil
def gather_age_key(cluster):
@@ -438,98 +435,6 @@
return False, f"Error while cleaning items: {e}"
-async def check_kustomization_ready(self, kustomization_name, namespace, timeout):
- self.logger.info("check_kustomization_ready Enter")
- self.logger.info(
- f"Kustomization {kustomization_name}. Namespace: {namespace}. Timeout: {timeout}"
- )
- counter = 1
- retry_time = self._odu_checkloop_retry_time
- max_iterations = ceil(timeout / retry_time)
- while counter <= max_iterations:
- kustomization = await self._kubectl.get_generic_object(
- api_group="kustomize.toolkit.fluxcd.io",
- api_plural="kustomizations",
- api_version="v1",
- namespace=namespace,
- name=kustomization_name,
- )
- if kustomization:
- # self.logger.debug(f"{yaml.safe_dump(kustomization)}")
- conditions = kustomization.get("status", {}).get("conditions", [])
- else:
- conditions = []
- self.logger.info(f"Kustomization status conditions: {conditions}")
- result = next((item for item in conditions if item["type"] == "Ready"), {})
- if result.get("status", "False") == "True":
- self.logger.info(
- f"Kustomization {kustomization_name} ready in {counter} iterations (aprox {counter*retry_time} seconds)"
- )
- return True, "COMPLETED"
- await asyncio.sleep(retry_time)
- counter += 1
- return (
- False,
- "Kustomization {kustomization_name} did not complete after {max_iterations} iterations (aprox {timeout} seconds)",
- )
-
-
-async def check_cluster_flag(self, cluster_name, cloud_type, flag, timeout):
- self.logger.info("check_cluster_flag Enter")
- self.logger.info(
- f"Cluster {cluster_name}. Cloud: {cloud_type}. Flag: {flag}. Timeout: {timeout}"
- )
- cluster_api_map = {
- "aws": {
- "api_group": "eks.aws.upbound.io",
- "api_plural": "clusters",
- "api_version": "v1beta1",
- },
- "azure": {
- "api_group": "containerservice.azure.upbound.io",
- "api_plural": "kubernetesclusters",
- "api_version": "v1beta1",
- },
- "gcp": {
- "api_group": "container.gcp.upbound.io",
- "api_plural": "clusters",
- "api_version": "v1beta2",
- },
- }
- api_group = cluster_api_map[cloud_type]["api_group"]
- api_plural = cluster_api_map[cloud_type]["api_plural"]
- api_version = cluster_api_map[cloud_type]["api_version"]
- counter = 1
- retry_time = self._odu_checkloop_retry_time
- max_iterations = ceil(timeout / retry_time)
- while counter <= max_iterations:
- managed_cluster = await self._kubectl.get_generic_object(
- api_group=api_group,
- api_plural=api_plural,
- api_version=api_version,
- namespace="",
- name=cluster_name,
- )
- if managed_cluster:
- # self.logger.debug(f"{yaml.safe_dump(managed_cluster)}")
- conditions = managed_cluster.get("status", {}).get("conditions", [])
- else:
- conditions = []
- self.logger.info(f"Cluster status conditions (flag {flag}): {conditions}")
- result = next((item for item in conditions if item["type"] == flag), {})
- if result.get("status", "False") == "True":
- self.logger.info(
- f"Cluster {cluster_name} reached the status {flag} in {counter} iterations (aprox {counter*retry_time} seconds)"
- )
- return True, "COMPLETED"
- await asyncio.sleep(retry_time)
- counter += 1
- return (
- False,
- "Cluster {cluster_name} did not reach the status {flag} after {max_iterations} seconds (aprox {timeout} seconds)",
- )
-
-
async def check_create_cluster(self, op_id, op_params, content):
self.logger.info(f"Operation {op_id}. Params: {op_params}. Content: {content}")
db_cluster = content["cluster"]
@@ -541,51 +446,41 @@
if cloud_type not in ("azure", "gcp", "aws"):
return False, "Not suitable VIM account to check cluster status"
- try:
- # Kustomization ready
- # kubectl get Kustomization/myakscluster01 -n managed-resources -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status'
- status, message = await self.check_kustomization_ready(
- kustomization_name=cluster_kustomization_name,
- namespace="managed-resources",
- timeout=self._odu_checkloop_kustomization_timeout,
- )
- if not status:
- return status, message
- # Cluster managed resource synced
- # kubectl get kubernetescluster.containerservice.azure.upbound.io/myakscluster01 -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Synced").status'
- status, message = await self.check_cluster_flag(
- cluster_name=cluster_name,
- cloud_type=cloud_type,
- flag="Synced",
- timeout=self._odu_checkloop_resource_timeout,
- )
- if not status:
- return status, message
- # Cluster managed resource ready
- # kubectl get kubernetescluster.containerservice.azure.upbound.io/myakscluster01 -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status'
- status, message = await self.check_cluster_flag(
- cluster_name=cluster_name,
- cloud_type=cloud_type,
- flag="Ready",
- timeout=self._odu_checkloop_resource_timeout,
- )
- if not status:
- return status, message
- # Bootstrap is complete
- # kubectl get Kustomization/myakscluster01-bstrp-fluxctrl -n managed-resources -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status'
- if bootstrap:
- status, message = await self.check_kustomization_ready(
- kustomization_name=f"{cluster_kustomization_name}-bstrp-fluxctrl",
- namespace="managed-resources",
- timeout=self._odu_checkloop_kustomization_timeout,
- )
- if not status:
- return status, message
- except Exception as e:
- self.logger.debug(traceback.format_exc())
- self.logger.debug(f"Exception: {e}", exc_info=True)
- return False, f"Unexpected exception: {e}"
- return True, "OK"
+ checkings_list = [
+ {
+ "item": "kustomization",
+ "name": cluster_kustomization_name,
+ "namespace": "managed-resources",
+ "flag": "Ready",
+ "timeout": self._odu_checkloop_kustomization_timeout,
+ "enable": True,
+ },
+ {
+ "item": f"cluster_{cloud_type}",
+ "name": cluster_name,
+ "namespace": "",
+ "flag": "Synced",
+ "timeout": self._odu_checkloop_resource_timeout,
+ "enable": True,
+ },
+ {
+ "item": f"cluster_{cloud_type}",
+ "name": cluster_name,
+ "namespace": "",
+ "flag": "Ready",
+ "timeout": self._odu_checkloop_resource_timeout,
+ "enable": True,
+ },
+ {
+ "item": "kustomization",
+ "name": f"{cluster_kustomization_name}-bstrp-fluxctrl",
+ "namespace": "managed-resources",
+ "flag": "Ready",
+ "timeout": self._odu_checkloop_kustomization_timeout,
+ "enable": bootstrap,
+ },
+ ]
+ return await self.common_check_list(checkings_list)
async def check_update_cluster(self, op_id, op_params, content):
diff --git a/osm_lcm/odu_libs/workflows.py b/osm_lcm/odu_libs/workflows.py
index 86cf743..a95e982 100644
--- a/osm_lcm/odu_libs/workflows.py
+++ b/osm_lcm/odu_libs/workflows.py
@@ -17,6 +17,7 @@
import asyncio
+import traceback
from math import ceil
@@ -101,3 +102,23 @@
False,
"{item} {name} was not ready after {max_iterations} iterations (aprox {timeout} seconds)",
)
+
+
+async def common_check_list(self, checkings_list):
+ try:
+ for checking in checkings_list:
+ if checking["enable"]:
+ status, message = await self.readiness_loop(
+ item=checking["item"],
+ name=checking["name"],
+ namespace=checking["namespace"],
+ flag=checking["flag"],
+ timeout=checking["timeout"],
+ )
+ if not status:
+ return status, message
+ except Exception as e:
+ self.logger.debug(traceback.format_exc())
+ self.logger.debug(f"Exception: {e}", exc_info=True)
+ return False, f"Unexpected exception: {e}"
+ return True, "OK"
diff --git a/osm_lcm/odu_workflows.py b/osm_lcm/odu_workflows.py
index e15f65e..a563f37 100644
--- a/osm_lcm/odu_workflows.py
+++ b/osm_lcm/odu_workflows.py
@@ -165,8 +165,6 @@
clean_items_cluster_create,
clean_items_cluster_update,
clean_items_cluster_register,
- check_kustomization_ready,
- check_cluster_flag,
check_create_cluster,
check_update_cluster,
check_delete_cluster,
@@ -209,6 +207,7 @@
from osm_lcm.odu_libs.workflows import (
check_workflow_status,
readiness_loop,
+ common_check_list,
)
from osm_lcm.odu_libs.render import (
render_jinja_template,