installers/install_kubeadm_cluster.sh

#!/bin/bash
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

set +eux

K8S_VERSION=1.28
K8S_PACKAGE_VERSION="$K8S_VERSION".2-1.1

# installs kubernetes packages
function install_kube() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    # Kubernetes releases can be found here: https://kubernetes.io/releases/
    # To check other available versions, run the following command
    # curl -s https://packages.cloud.google.com/apt/dists/kubernetes-xenial/main/binary-amd64/Packages | grep Version | awk '{print $2}'
    sudo apt-get -y update && sudo apt-get install -y apt-transport-https ca-certificates curl
    curl -fsSL https://pkgs.k8s.io/core:/stable:/v"$K8S_VERSION"/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
    echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$K8S_VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
    sudo apt-get -y update
    echo "Installing Kubernetes Packages ..."
    sudo apt-get install -y kubelet=${K8S_PACKAGE_VERSION} kubeadm=${K8S_PACKAGE_VERSION} kubectl=${K8S_PACKAGE_VERSION}
    sudo apt-mark hold kubelet kubeadm kubectl
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# check and track kube packages installation
function check_and_track_kube_install() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    kubelet_version=$(dpkg -s kubelet|grep Version|awk '{print $2}')
    [ -n "${kubelet_version}" ] || FATAL_TRACK k8scluster "Kubelet was not installed."
    kubeadm_version=$(dpkg -s kubeadm|grep Version|awk '{print $2}')
    [ -n "${kubeadm_version}" ] || FATAL_TRACK k8scluster "Kubeadm was not installed."
    kubectl_version=$(dpkg -s kubectl|grep Version|awk '{print $2}')
    [ -n "${kubectl_version}" ] || FATAL_TRACK k8scluster "Kubectl was not installed."
    track k8scluster install_k8s_ok none none none kubelet ${kubelet_version} none none kubeadm ${kubeadm_version} none none kubectl ${kubectl_version} none none
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# initializes kubernetes control plane
function init_kubeadm() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    sudo modprobe br_netfilter
    echo "br_netfilter" | sudo tee /etc/modules-load.d/k8s.conf
    sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
    echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee /etc/sysctl.d/k8s.conf
    sudo swapoff -a
    sudo sed -i.bak '/.*none.*swap/s/^\(.*\)$/#\1/g' /etc/fstab
    sudo kubeadm init --config $1 --dry-run || FATAL_TRACK k8scluster "kubeadm init dry-run failed"
    sudo kubeadm init --config $1
    sleep 5
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# Initializes kubeconfig file
function kube_config_dir() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    K8S_MANIFEST_DIR="/etc/kubernetes/manifests"
    [ ! -d $K8S_MANIFEST_DIR ] && FATAL_TRACK k8scluster "Kubernetes folder $K8S_MANIFEST_DIR was not found"
    mkdir -p $HOME/.kube
    sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# test kubernetes installation
function check_and_track_init_k8s() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    echo "Reading existing namespaces"
    kubectl get ns || FATAL_TRACK k8scluster "Failed getting namespaces"
    track k8scluster init_k8s_ok
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# deploys flannel as daemonsets
function deploy_cni_provider() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    CNI_DIR="$(mktemp -d -q --tmpdir "flannel.XXXXXX")"
    trap 'rm -rf "${CNI_DIR}"' EXIT
    KUBE_FLANNEL_FILE_URL="https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"
    wget --retry-on-host-error --retry-on-http-error 404,429,503 --tries=5 "${KUBE_FLANNEL_FILE_URL}" -P $CNI_DIR
    [ ! -f $CNI_DIR/kube-flannel.yml ] && FATAL_TRACK k8scluster "Cannot Install Flannel because $CNI_DIR/kube-flannel.yml was not found. Maybe the file ${KUBE_FLANNEL_FILE_URL} is temporarily not accessible"
    kubectl apply -f $CNI_DIR
    [ $? -ne 0 ] && FATAL_TRACK k8scluster "Cannot Install Flannel"
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# taints K8s master node
function taint_master_node() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    K8S_MASTER=$(kubectl get nodes | awk '$3~/control-plane/'| awk '{print $1; exit}')
    kubectl taint node $K8S_MASTER node-role.kubernetes.io/control-plane:NoSchedule-
    sleep 5
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# check and track kube packages installation
function check_and_track_k8s_ready_before_helm() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    kubectl get events || FATAL_TRACK k8scluster "Failed getting events"
    track k8scluster k8s_ready_before_helm
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

#Install Helm v3
#Helm releases can be found here: https://github.com/helm/helm/releases
function install_helm() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    HELM_VERSION="v3.11.3"
    if ! [[ "$(helm version --short 2>/dev/null)" =~ ^v3.* ]]; then
        # Helm is not installed. Install helm
        echo "Helm3 is not installed, installing ..."
        curl https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz --output helm-${HELM_VERSION}.tar.gz
        tar -zxvf helm-${HELM_VERSION}.tar.gz
        sudo mv linux-amd64/helm /usr/local/bin/helm
        rm -r linux-amd64
        rm helm-${HELM_VERSION}.tar.gz
    else
        echo "Helm3 is already installed. Skipping installation..."
    fi
    helm version || FATAL_TRACK k8scluster "Could not obtain helm version. Maybe helm client was not installed"
    helm repo add stable https://charts.helm.sh/stable || FATAL_TRACK k8scluster "Helm repo stable could not be added"
    helm repo update || FATAL_TRACK k8scluster "Helm repo stable could not be updated"
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

function install_k8s_storageclass() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    # Openebs versions can be found here: https://github.com/openebs/openebs/releases
    OPENEBS_VERSION="3.7.0"
    echo "Installing OpenEBS"
    helm repo add openebs https://openebs.github.io/charts
    helm repo update
    helm install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION}
    helm ls -n openebs
    local storageclass_timeout=400
    local counter=0
    local storageclass_ready=""
    echo "Waiting for storageclass"
    while (( counter < storageclass_timeout ))
    do
        kubectl get storageclass openebs-hostpath &> /dev/null

        if [ $? -eq 0 ] ; then
            echo "Storageclass available"
            storageclass_ready="y"
            break
        else
            counter=$((counter + 15))
            sleep 15
        fi
    done
    [ -n "$storageclass_ready" ] || FATAL_TRACK k8scluster "Storageclass not ready after $storageclass_timeout seconds. Cannot install openebs"
    kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

#installs metallb from helm
function install_helm_metallb() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    echo "Installing MetalLB"
    METALLB_VERSION="0.13.10"
    helm repo add metallb https://metallb.github.io/metallb
    helm repo update
    helm install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION}
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

function configure_ipaddresspool_metallb() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    echo "Creating IP address pool manifest: ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
    METALLB_IP_RANGE="$DEFAULT_IP/32"
    echo "apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: first-pool
  namespace: metallb-system
spec:
  addresses:
  - ${METALLB_IP_RANGE}" | sudo tee -a ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml
    echo "Applying IP address pool manifest: kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
    kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml || FATAL_TRACK k8scluster "Cannot create IP address Pool in MetalLB"
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

#installs cert-manager
function install_helm_certmanager() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    echo "Installing cert-manager"
    CERTMANAGER_VERSION="v1.9.1"
    helm repo add jetstack https://charts.jetstack.io
    helm repo update
    helm install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \
        --version ${CERTMANAGER_VERSION} --set installCRDs=true --set prometheus.enabled=false \
        --set clusterResourceNamespace=osm \
        --set extraArgs="{--enable-certificate-owner-ref=true}"
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

#checks openebs and metallb readiness
function check_for_readiness() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    # Default input values
    sampling_period=2       # seconds
    time_for_readiness=20   # seconds ready
    time_for_failure=200    # seconds broken
    OPENEBS_NAMESPACE=openebs
    METALLB_NAMESPACE=metallb-system
    # STACK_NAME=osm          # By default, "osm"

    # Equivalent number of samples
    oks_threshold=$((time_for_readiness/${sampling_period}))     # No. ok samples to declare the system ready
    failures_threshold=$((time_for_failure/${sampling_period}))  # No. nok samples to declare the system broken
    failures_in_a_row=0
    oks_in_a_row=0

    ####################################################################################
    # Loop to check system readiness
    ####################################################################################
    while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
    do
        # State of OpenEBS
        OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1)
        OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
        OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
        COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l)
        COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l)

        # State of MetalLB
        METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1)
        METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
        METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
        COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l)
        COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l)

        # State of CertManager
        CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1)
        CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
        CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
        COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l)
        COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l)

        # OK sample
        if [[ $((${COUNT_OPENEBS_NOT_READY}+${COUNT_METALLB_NOT_READY})) -eq 0 ]]
        then
            ((++oks_in_a_row))
            failures_in_a_row=0
            echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r
        # NOK sample
        else
            ((++failures_in_a_row))
            oks_in_a_row=0
            echo
            echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold}

            # Reports failed pods in OpenEBS
            if [[ "${COUNT_OPENEBS_NOT_READY}" -ne 0 ]]
            then
                echo "OpenEBS: Waiting for ${COUNT_OPENEBS_NOT_READY} of $((${COUNT_OPENEBS_NOT_READY}+${COUNT_OPENEBS_READY})) pods to be ready:"
                echo "${OPENEBS_NOT_READY}"
                echo
            fi

            # Reports failed pods in MetalLB
            if [[ "${COUNT_METALLB_NOT_READY}" -ne 0 ]]
            then
                echo "MetalLB: Waiting for ${COUNT_METALLB_NOT_READY} of $((${COUNT_METALLB_NOT_READY}+${COUNT_METALLB_READY})) pods to be ready:"
                echo "${METALLB_NOT_READY}"
                echo
            fi

            # Reports failed pods in CertManager
            if [[ "${COUNT_CERTMANAGER_NOT_READY}" -ne 0 ]]
            then
                echo "CertManager: Waiting for ${COUNT_CERTMANAGER_NOT_READY} of $((${COUNT_CERTMANAGER_NOT_READY}+${COUNT_CERTMANAGER_READY})) pods to be ready:"
                echo "${CERTMANAGER_NOT_READY}"
                echo
            fi
        fi

        #------------ NEXT SAMPLE
        sleep ${sampling_period}
    done

    ####################################################################################
    # OUTCOME
    ####################################################################################
    if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
    then
        echo
        FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN"
    else
        echo
        echo "K8S CLUSTER IS READY"
    fi
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

#removes osm deployments and services
function remove_k8s_namespace() {
    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
    echo "Deleting existing namespace $1: kubectl delete ns $1"
    kubectl delete ns $1 2>/dev/null
    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}

# main
while getopts ":D:d:i:-: " o; do
    case "${o}" in
        i)
            DEFAULT_IP="${OPTARG}"
            ;;
        d)
            OSM_CLUSTER_WORK_DIR="${OPTARG}"
            ;;
        D)
            OSM_DEVOPS="${OPTARG}"
            ;;
        -)
            [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
            echo -e "Invalid option: '--$OPTARG'\n" >&2
            exit 1
            ;;
        :)
            echo "Option -$OPTARG requires an argument" >&2
            exit 1
            ;;
        \?)
            echo -e "Invalid option: '-$OPTARG'\n" >&2
            exit 1
            ;;
        *)
            exit 1
            ;;
    esac
done

source $OSM_DEVOPS/common/logging
source $OSM_DEVOPS/common/track

echo "DEBUG_INSTALL=$DEBUG_INSTALL"
echo "DEFAULT_IP=$DEFAULT_IP"
echo "OSM_DEVOPS=$OSM_DEVOPS"
echo "OSM_CLUSTER_WORK_DIR=$OSM_CLUSTER_WORK_DIR"
echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR"
echo "HOME=$HOME"


install_kube
check_and_track_kube_install

init_kubeadm $OSM_CLUSTER_WORK_DIR/kubeadm-config.yaml
kube_config_dir
check_and_track_init_k8s

if [ -n "$INSTALL_K8S_MONITOR" ]; then
    # uninstall OSM MONITORING
    uninstall_k8s_monitoring
    track k8scluster uninstall_k8s_monitoring_ok
fi

deploy_cni_provider
taint_master_node
check_and_track_k8s_ready_before_helm

remove_k8s_namespace osm

install_helm
track k8scluster install_helm_ok

install_k8s_storageclass
track k8scluster k8s_storageclass_ok
install_helm_metallb
track k8scluster k8s_metallb_ok
install_helm_certmanager
track k8scluster k8s_certmanager_ok
check_for_readiness
track k8scluster k8s_ready_ok
configure_ipaddresspool_metallb