From 41f5ce501984dcb38c25822f26d4c329d0983308 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Mon, 1 Apr 2024 17:46:09 +0200 Subject: [PATCH] Separate cluster addons installation (openebs, metallb, cert-mgr) from install_kubeadm_cluster Change-Id: Id80d48b804372c97290f3cb4ac930ca76f1d692b Signed-off-by: garciadeblas --- installers/full_install_osm.sh | 5 +- installers/install_cluster_addons.sh | 238 ++++++++++++++++++++++++++ installers/install_kubeadm_cluster.sh | 13 +- 3 files changed, 244 insertions(+), 12 deletions(-) create mode 100755 installers/install_cluster_addons.sh diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index b25cdaa8..b7df2d5c 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -508,9 +508,12 @@ function install_osm() { track helm_client install_helm_client_ok if [ "${K8S_CLUSTER_ENGINE}" == "kubeadm" ]; then - KUBEADM_INSTALL_OPTS="-i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}" + KUBEADM_INSTALL_OPTS="-d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}" $OSM_DEVOPS/installers/install_kubeadm_cluster.sh ${KUBEADM_INSTALL_OPTS} || \ FATAL_TRACK k8scluster "install_kubeadm_cluster.sh failed" + K8SCLUSTER_ADDONS_INSTALL_OPTS="-i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}" + $OSM_DEVOPS/installers/install_cluster_addons.sh ${K8SCLUSTER_ADDONS_INSTALL_OPTS} || \ + FATAL_TRACK k8scluster "install_cluster_addons.sh failed" fi track k8scluster k8scluster_ok diff --git a/installers/install_cluster_addons.sh b/installers/install_cluster_addons.sh new file mode 100755 index 00000000..a2bc366a --- /dev/null +++ b/installers/install_cluster_addons.sh @@ -0,0 +1,238 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +eux + +function install_k8s_storageclass() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + # Openebs versions can be found here: https://github.com/openebs/openebs/releases + OPENEBS_VERSION="3.7.0" + echo "Installing OpenEBS" + helm repo add openebs https://openebs.github.io/charts + helm repo update + helm install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION} + helm ls -n openebs + local storageclass_timeout=400 + local counter=0 + local storageclass_ready="" + echo "Waiting for storageclass" + while (( counter < storageclass_timeout )) + do + kubectl get storageclass openebs-hostpath &> /dev/null + + if [ $? -eq 0 ] ; then + echo "Storageclass available" + storageclass_ready="y" + break + else + counter=$((counter + 15)) + sleep 15 + fi + done + [ -n "$storageclass_ready" ] || FATAL_TRACK k8scluster "Storageclass not ready after $storageclass_timeout seconds. Cannot install openebs" + kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +#installs metallb from helm +function install_helm_metallb() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + echo "Installing MetalLB" + METALLB_VERSION="0.13.10" + helm repo add metallb https://metallb.github.io/metallb + helm repo update + helm install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION} + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +function configure_ipaddresspool_metallb() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + echo "Creating IP address pool manifest: ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml" + METALLB_IP_RANGE="$DEFAULT_IP/32" + echo "apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: first-pool + namespace: metallb-system +spec: + addresses: + - ${METALLB_IP_RANGE}" | sudo tee -a ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml + echo "Applying IP address pool manifest: kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml" + kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml || FATAL_TRACK k8scluster "Cannot create IP address Pool in MetalLB" + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +#installs cert-manager +function install_helm_certmanager() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + echo "Installing cert-manager" + CERTMANAGER_VERSION="v1.9.1" + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \ + --version ${CERTMANAGER_VERSION} --set installCRDs=true --set prometheus.enabled=false \ + --set clusterResourceNamespace=osm \ + --set extraArgs="{--enable-certificate-owner-ref=true}" + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +#checks openebs and metallb readiness +function check_for_readiness() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + # Default input values + sampling_period=2 # seconds + time_for_readiness=20 # seconds ready + time_for_failure=200 # seconds broken + OPENEBS_NAMESPACE=openebs + METALLB_NAMESPACE=metallb-system + CERTMANAGER_NAMESPACE=cert-manager + # STACK_NAME=osm # By default, "osm" + + # Equivalent number of samples + oks_threshold=$((time_for_readiness/${sampling_period})) # No. ok samples to declare the system ready + failures_threshold=$((time_for_failure/${sampling_period})) # No. nok samples to declare the system broken + failures_in_a_row=0 + oks_in_a_row=0 + + #################################################################################### + # Loop to check system readiness + #################################################################################### + while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]] + do + # State of OpenEBS + OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1) + OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l) + COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l) + + # State of MetalLB + METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1) + METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}') + METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l) + COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l) + + # State of CertManager + CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1) + CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l) + COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l) + + # OK sample + if [[ $((${COUNT_OPENEBS_NOT_READY}+${COUNT_METALLB_NOT_READY})) -eq 0 ]] + then + ((++oks_in_a_row)) + failures_in_a_row=0 + echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r + # NOK sample + else + ((++failures_in_a_row)) + oks_in_a_row=0 + echo + echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold} + + # Reports failed pods in OpenEBS + if [[ "${COUNT_OPENEBS_NOT_READY}" -ne 0 ]] + then + echo "OpenEBS: Waiting for ${COUNT_OPENEBS_NOT_READY} of $((${COUNT_OPENEBS_NOT_READY}+${COUNT_OPENEBS_READY})) pods to be ready:" + echo "${OPENEBS_NOT_READY}" + echo + fi + + # Reports failed pods in MetalLB + if [[ "${COUNT_METALLB_NOT_READY}" -ne 0 ]] + then + echo "MetalLB: Waiting for ${COUNT_METALLB_NOT_READY} of $((${COUNT_METALLB_NOT_READY}+${COUNT_METALLB_READY})) pods to be ready:" + echo "${METALLB_NOT_READY}" + echo + fi + + # Reports failed pods in CertManager + if [[ "${COUNT_CERTMANAGER_NOT_READY}" -ne 0 ]] + then + echo "CertManager: Waiting for ${COUNT_CERTMANAGER_NOT_READY} of $((${COUNT_CERTMANAGER_NOT_READY}+${COUNT_CERTMANAGER_READY})) pods to be ready:" + echo "${CERTMANAGER_NOT_READY}" + echo + fi + fi + + #------------ NEXT SAMPLE + sleep ${sampling_period} + done + + #################################################################################### + # OUTCOME + #################################################################################### + if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]] + then + echo + FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN" + else + echo + echo "K8S CLUSTER IS READY" + fi + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# main +while getopts ":D:d:i:-: " o; do + case "${o}" in + i) + DEFAULT_IP="${OPTARG}" + ;; + d) + OSM_CLUSTER_WORK_DIR="${OPTARG}" + ;; + D) + OSM_DEVOPS="${OPTARG}" + ;; + -) + [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue + echo -e "Invalid option: '--$OPTARG'\n" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument" >&2 + exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\n" >&2 + exit 1 + ;; + *) + exit 1 + ;; + esac +done + +source $OSM_DEVOPS/common/logging +source $OSM_DEVOPS/common/track + +echo "DEBUG_INSTALL=$DEBUG_INSTALL" +echo "DEFAULT_IP=$DEFAULT_IP" +echo "OSM_DEVOPS=$OSM_DEVOPS" +echo "OSM_CLUSTER_WORK_DIR=$OSM_CLUSTER_WORK_DIR" + +install_k8s_storageclass +track k8scluster k8s_storageclass_ok +install_helm_metallb +track k8scluster k8s_metallb_ok +install_helm_certmanager +track k8scluster k8s_certmanager_ok +check_for_readiness +track k8scluster k8s_ready_ok +configure_ipaddresspool_metallb diff --git a/installers/install_kubeadm_cluster.sh b/installers/install_kubeadm_cluster.sh index 51e815aa..cb0b22de 100755 --- a/installers/install_kubeadm_cluster.sh +++ b/installers/install_kubeadm_cluster.sh @@ -324,10 +324,8 @@ source $OSM_DEVOPS/common/logging source $OSM_DEVOPS/common/track echo "DEBUG_INSTALL=$DEBUG_INSTALL" -echo "DEFAULT_IP=$DEFAULT_IP" echo "OSM_DEVOPS=$OSM_DEVOPS" echo "OSM_CLUSTER_WORK_DIR=$OSM_CLUSTER_WORK_DIR" -echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR" echo "HOME=$HOME" echo "Creating folders for installation" @@ -352,12 +350,5 @@ remove_k8s_namespace osm # but tracking is still here because the installation analytics still expects it track k8scluster install_helm_ok -install_k8s_storageclass -track k8scluster k8s_storageclass_ok -install_helm_metallb -track k8scluster k8s_metallb_ok -install_helm_certmanager -track k8scluster k8s_certmanager_ok -check_for_readiness -track k8scluster k8s_ready_ok -configure_ipaddresspool_metallb +# Installation of storage class, metallb and cert-manager has been moved +# to install_cluster_addons.sh, run from full_install_osm.sh -- 2.25.1