From 29d041a2620860b1add7729787ba91ec99009107 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Tue, 22 Oct 2024 12:54:22 +0200 Subject: [PATCH] Refactor cluster checks to use common_check_list and readiness_loop Change-Id: I3a1b28b7139c863805b5d66a461dc4a3a6baded8 Signed-off-by: garciadeblas --- osm_lcm/odu_libs/cluster_mgmt.py | 175 +++++++------------------------ osm_lcm/odu_libs/workflows.py | 21 ++++ osm_lcm/odu_workflows.py | 3 +- 3 files changed, 57 insertions(+), 142 deletions(-) diff --git a/osm_lcm/odu_libs/cluster_mgmt.py b/osm_lcm/odu_libs/cluster_mgmt.py index a86ce20b..1c216750 100644 --- a/osm_lcm/odu_libs/cluster_mgmt.py +++ b/osm_lcm/odu_libs/cluster_mgmt.py @@ -17,11 +17,8 @@ from pyrage import x25519 -import asyncio -import traceback import yaml import base64 -from math import ceil def gather_age_key(cluster): @@ -438,98 +435,6 @@ async def clean_items_cluster_register(self, op_id, op_params, content): return False, f"Error while cleaning items: {e}" -async def check_kustomization_ready(self, kustomization_name, namespace, timeout): - self.logger.info("check_kustomization_ready Enter") - self.logger.info( - f"Kustomization {kustomization_name}. Namespace: {namespace}. Timeout: {timeout}" - ) - counter = 1 - retry_time = self._odu_checkloop_retry_time - max_iterations = ceil(timeout / retry_time) - while counter <= max_iterations: - kustomization = await self._kubectl.get_generic_object( - api_group="kustomize.toolkit.fluxcd.io", - api_plural="kustomizations", - api_version="v1", - namespace=namespace, - name=kustomization_name, - ) - if kustomization: - # self.logger.debug(f"{yaml.safe_dump(kustomization)}") - conditions = kustomization.get("status", {}).get("conditions", []) - else: - conditions = [] - self.logger.info(f"Kustomization status conditions: {conditions}") - result = next((item for item in conditions if item["type"] == "Ready"), {}) - if result.get("status", "False") == "True": - self.logger.info( - f"Kustomization {kustomization_name} ready in {counter} iterations (aprox {counter*retry_time} seconds)" - ) - return True, "COMPLETED" - await asyncio.sleep(retry_time) - counter += 1 - return ( - False, - "Kustomization {kustomization_name} did not complete after {max_iterations} iterations (aprox {timeout} seconds)", - ) - - -async def check_cluster_flag(self, cluster_name, cloud_type, flag, timeout): - self.logger.info("check_cluster_flag Enter") - self.logger.info( - f"Cluster {cluster_name}. Cloud: {cloud_type}. Flag: {flag}. Timeout: {timeout}" - ) - cluster_api_map = { - "aws": { - "api_group": "eks.aws.upbound.io", - "api_plural": "clusters", - "api_version": "v1beta1", - }, - "azure": { - "api_group": "containerservice.azure.upbound.io", - "api_plural": "kubernetesclusters", - "api_version": "v1beta1", - }, - "gcp": { - "api_group": "container.gcp.upbound.io", - "api_plural": "clusters", - "api_version": "v1beta2", - }, - } - api_group = cluster_api_map[cloud_type]["api_group"] - api_plural = cluster_api_map[cloud_type]["api_plural"] - api_version = cluster_api_map[cloud_type]["api_version"] - counter = 1 - retry_time = self._odu_checkloop_retry_time - max_iterations = ceil(timeout / retry_time) - while counter <= max_iterations: - managed_cluster = await self._kubectl.get_generic_object( - api_group=api_group, - api_plural=api_plural, - api_version=api_version, - namespace="", - name=cluster_name, - ) - if managed_cluster: - # self.logger.debug(f"{yaml.safe_dump(managed_cluster)}") - conditions = managed_cluster.get("status", {}).get("conditions", []) - else: - conditions = [] - self.logger.info(f"Cluster status conditions (flag {flag}): {conditions}") - result = next((item for item in conditions if item["type"] == flag), {}) - if result.get("status", "False") == "True": - self.logger.info( - f"Cluster {cluster_name} reached the status {flag} in {counter} iterations (aprox {counter*retry_time} seconds)" - ) - return True, "COMPLETED" - await asyncio.sleep(retry_time) - counter += 1 - return ( - False, - "Cluster {cluster_name} did not reach the status {flag} after {max_iterations} seconds (aprox {timeout} seconds)", - ) - - async def check_create_cluster(self, op_id, op_params, content): self.logger.info(f"Operation {op_id}. Params: {op_params}. Content: {content}") db_cluster = content["cluster"] @@ -541,51 +446,41 @@ async def check_create_cluster(self, op_id, op_params, content): if cloud_type not in ("azure", "gcp", "aws"): return False, "Not suitable VIM account to check cluster status" - try: - # Kustomization ready - # kubectl get Kustomization/myakscluster01 -n managed-resources -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status' - status, message = await self.check_kustomization_ready( - kustomization_name=cluster_kustomization_name, - namespace="managed-resources", - timeout=self._odu_checkloop_kustomization_timeout, - ) - if not status: - return status, message - # Cluster managed resource synced - # kubectl get kubernetescluster.containerservice.azure.upbound.io/myakscluster01 -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Synced").status' - status, message = await self.check_cluster_flag( - cluster_name=cluster_name, - cloud_type=cloud_type, - flag="Synced", - timeout=self._odu_checkloop_resource_timeout, - ) - if not status: - return status, message - # Cluster managed resource ready - # kubectl get kubernetescluster.containerservice.azure.upbound.io/myakscluster01 -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status' - status, message = await self.check_cluster_flag( - cluster_name=cluster_name, - cloud_type=cloud_type, - flag="Ready", - timeout=self._odu_checkloop_resource_timeout, - ) - if not status: - return status, message - # Bootstrap is complete - # kubectl get Kustomization/myakscluster01-bstrp-fluxctrl -n managed-resources -o jsonpath='{.status.conditions}' | jq -r '.[] | select(.type=="Ready").status' - if bootstrap: - status, message = await self.check_kustomization_ready( - kustomization_name=f"{cluster_kustomization_name}-bstrp-fluxctrl", - namespace="managed-resources", - timeout=self._odu_checkloop_kustomization_timeout, - ) - if not status: - return status, message - except Exception as e: - self.logger.debug(traceback.format_exc()) - self.logger.debug(f"Exception: {e}", exc_info=True) - return False, f"Unexpected exception: {e}" - return True, "OK" + checkings_list = [ + { + "item": "kustomization", + "name": cluster_kustomization_name, + "namespace": "managed-resources", + "flag": "Ready", + "timeout": self._odu_checkloop_kustomization_timeout, + "enable": True, + }, + { + "item": f"cluster_{cloud_type}", + "name": cluster_name, + "namespace": "", + "flag": "Synced", + "timeout": self._odu_checkloop_resource_timeout, + "enable": True, + }, + { + "item": f"cluster_{cloud_type}", + "name": cluster_name, + "namespace": "", + "flag": "Ready", + "timeout": self._odu_checkloop_resource_timeout, + "enable": True, + }, + { + "item": "kustomization", + "name": f"{cluster_kustomization_name}-bstrp-fluxctrl", + "namespace": "managed-resources", + "flag": "Ready", + "timeout": self._odu_checkloop_kustomization_timeout, + "enable": bootstrap, + }, + ] + return await self.common_check_list(checkings_list) async def check_update_cluster(self, op_id, op_params, content): diff --git a/osm_lcm/odu_libs/workflows.py b/osm_lcm/odu_libs/workflows.py index 86cf7437..a95e982a 100644 --- a/osm_lcm/odu_libs/workflows.py +++ b/osm_lcm/odu_libs/workflows.py @@ -17,6 +17,7 @@ import asyncio +import traceback from math import ceil @@ -101,3 +102,23 @@ async def readiness_loop(self, item, name, namespace, flag, timeout): False, "{item} {name} was not ready after {max_iterations} iterations (aprox {timeout} seconds)", ) + + +async def common_check_list(self, checkings_list): + try: + for checking in checkings_list: + if checking["enable"]: + status, message = await self.readiness_loop( + item=checking["item"], + name=checking["name"], + namespace=checking["namespace"], + flag=checking["flag"], + timeout=checking["timeout"], + ) + if not status: + return status, message + except Exception as e: + self.logger.debug(traceback.format_exc()) + self.logger.debug(f"Exception: {e}", exc_info=True) + return False, f"Unexpected exception: {e}" + return True, "OK" diff --git a/osm_lcm/odu_workflows.py b/osm_lcm/odu_workflows.py index e15f65ed..a563f379 100644 --- a/osm_lcm/odu_workflows.py +++ b/osm_lcm/odu_workflows.py @@ -165,8 +165,6 @@ class OduWorkflow(LcmBase): clean_items_cluster_create, clean_items_cluster_update, clean_items_cluster_register, - check_kustomization_ready, - check_cluster_flag, check_create_cluster, check_update_cluster, check_delete_cluster, @@ -209,6 +207,7 @@ class OduWorkflow(LcmBase): from osm_lcm.odu_libs.workflows import ( check_workflow_status, readiness_loop, + common_check_list, ) from osm_lcm.odu_libs.render import ( render_jinja_template, -- 2.25.1