installers/install_cluster_addons.sh

   1 #!/bin/bash
   2 #
   3 #   Licensed under the Apache License, Version 2.0 (the "License");
   4 #   you may not use this file except in compliance with the License.
   5 #   You may obtain a copy of the License at
   6 #
   7 #       http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #   Unless required by applicable law or agreed to in writing, software
  10 #   distributed under the License is distributed on an "AS IS" BASIS,
  11 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #   See the License for the specific language governing permissions and
  13 #   limitations under the License.
  14 #
  15
  16 set +eux
  17
  18 INSTALL_STORAGECLASS=""
  19 INSTALL_METALLB=""
  20 INSTALL_CERTMANAGER=""
  21 INSTALL_NGINX=""
  22
  23 function install_k8s_storageclass() {
  24     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  25     # Openebs versions can be found here: https://github.com/openebs/openebs/releases
  26     OPENEBS_VERSION="3.7.0"
  27     echo "Installing OpenEBS"
  28     helm repo add openebs https://openebs.github.io/charts
  29     helm repo update
  30     helm upgrade --install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION}
  31     helm ls -n openebs
  32     local storageclass_timeout=400
  33     local counter=0
  34     local storageclass_ready=""
  35     echo "Waiting for storageclass"
  36     while (( counter < storageclass_timeout ))
  37     do
  38         kubectl get storageclass openebs-hostpath &> /dev/null
  39
  40         if [ $? -eq 0 ] ; then
  41             echo "Storageclass available"
  42             storageclass_ready="y"
  43             break
  44         else
  45             counter=$((counter + 15))
  46             sleep 15
  47         fi
  48     done
  49     [ -n "$storageclass_ready" ] || FATAL_TRACK k8scluster "Storageclass not ready after $storageclass_timeout seconds. Cannot install openebs"
  50     kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
  51     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  52 }
  53
  54 #installs metallb from helm
  55 function install_helm_metallb() {
  56     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  57     echo "Installing MetalLB"
  58     METALLB_VERSION="0.13.10"
  59     helm repo add metallb https://metallb.github.io/metallb
  60     helm repo update
  61     # kubectl create namespace metallb-system
  62     # kubectl label namespaces metallb-system pod-security.kubernetes.io/enforce=privileged
  63     # kubectl label namespaces metallb-system pod-security.kubernetes.io/audit=privileged
  64     # kubectl label namespaces metallb-system pod-security.kubernetes.io/warn=privileged
  65     helm upgrade --install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION}
  66     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  67 }
  68
  69 function configure_ipaddresspool_metallb() {
  70     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  71     echo "Creating IP address pool manifest: ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
  72     [ ! -d "$OSM_CLUSTER_WORK_DIR" ] && sudo mkdir -p $OSM_CLUSTER_WORK_DIR
  73     METALLB_IP_RANGE="$DEFAULT_IP/32"
  74     echo "apiVersion: metallb.io/v1beta1
  75 kind: IPAddressPool
  76 metadata:
  77   name: first-pool
  78   namespace: metallb-system
  79 spec:
  80   addresses:
  81   - ${METALLB_IP_RANGE}" | sudo tee -a ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml
  82     echo "Applying IP address pool manifest: kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
  83     kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml || FATAL_TRACK k8scluster "Cannot create IP address Pool in MetalLB"
  84     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  85 }
  86
  87 #installs cert-manager
  88 function install_helm_certmanager() {
  89     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  90     echo "Installing cert-manager"
  91     CERTMANAGER_VERSION="v1.9.1"
  92     helm repo add jetstack https://charts.jetstack.io
  93     helm repo update
  94     helm upgrade --install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \
  95         --version ${CERTMANAGER_VERSION} --set installCRDs=true --set prometheus.enabled=false \
  96         --set clusterResourceNamespace=osm \
  97         --set extraArgs="{--enable-certificate-owner-ref=true}"
  98     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  99 }
 100
 101 #installs nginx
 102 function install_helm_nginx() {
 103     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
 104     echo "Installing nginx"
 105     NGINX_VERSION="4.10.0"
 106     ANNOTATIONS='--set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz'
 107     ANNOTATIONS=${ANNOTATIONS:-""}
 108     helm upgrade --install ingress-nginx ingress-nginx \
 109         --repo https://kubernetes.github.io/ingress-nginx --version ${NGINX_VERSION} \
 110         --namespace ingress-nginx --create-namespace ${ANNOTATIONS}
 111     # Wait until ready
 112     kubectl wait --namespace ingress-nginx \
 113         --for=condition=ready pod \
 114         --selector=app.kubernetes.io/component=controller \
 115         --timeout=120s
 116     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 117 }
 118
 119 #checks openebs, metallb and cert-manager readiness
 120 function check_for_readiness() {
 121     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
 122     # Default input values
 123     sampling_period=2       # seconds
 124     time_for_readiness=20   # seconds ready
 125     time_for_failure=200    # seconds broken
 126     OPENEBS_NAMESPACE=openebs
 127     METALLB_NAMESPACE=metallb-system
 128     CERTMANAGER_NAMESPACE=cert-manager
 129
 130     # Equivalent number of samples
 131     oks_threshold=$((time_for_readiness/${sampling_period}))     # No. ok samples to declare the system ready
 132     failures_threshold=$((time_for_failure/${sampling_period}))  # No. nok samples to declare the system broken
 133     failures_in_a_row=0
 134     oks_in_a_row=0
 135
 136     ####################################################################################
 137     # Loop to check system readiness
 138     ####################################################################################
 139     while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
 140     do
 141         # State of OpenEBS
 142         if [ -n "${INSTALL_STORAGECLASS}" ]; then
 143             OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1)
 144             OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 145             OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 146             COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l)
 147             COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l)
 148         fi
 149
 150         # State of MetalLB
 151         if [ -n "${INSTALL_METALLB}" ]; then
 152             METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1)
 153             METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
 154             METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
 155             COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l)
 156             COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l)
 157         fi
 158
 159         # State of CertManager
 160         if [ -n "${INSTALL_CERTMANAGER}" ]; then
 161             CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1)
 162             CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 163             CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 164             COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l)
 165             COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l)
 166         fi
 167
 168         # OK sample
 169         if [[ $((${COUNT_OPENEBS_NOT_READY:-0}+${COUNT_METALLB_NOT_READY:-0}+${COUNT_CERTMANAGER_NOT_READY:-0})) -eq 0 ]]
 170         then
 171             ((++oks_in_a_row))
 172             failures_in_a_row=0
 173             echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r
 174         # NOK sample
 175         else
 176             ((++failures_in_a_row))
 177             oks_in_a_row=0
 178             echo
 179             echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold}
 180
 181             # Reports failed pods in OpenEBS
 182             if [[ "${COUNT_OPENEBS_NOT_READY:-0}" -ne 0 ]]
 183             then
 184                 echo "OpenEBS: Waiting for ${COUNT_OPENEBS_NOT_READY} of $((${COUNT_OPENEBS_NOT_READY}+${COUNT_OPENEBS_READY})) pods to be ready:"
 185                 echo "${OPENEBS_NOT_READY}"
 186                 echo
 187             fi
 188
 189             # Reports failed pods in MetalLB
 190             if [[ "${COUNT_METALLB_NOT_READY:-0}" -ne 0 ]]
 191             then
 192                 echo "MetalLB: Waiting for ${COUNT_METALLB_NOT_READY} of $((${COUNT_METALLB_NOT_READY}+${COUNT_METALLB_READY})) pods to be ready:"
 193                 echo "${METALLB_NOT_READY}"
 194                 echo
 195             fi
 196
 197             # Reports failed pods in CertManager
 198             if [[ "${COUNT_CERTMANAGER_NOT_READY:-0}" -ne 0 ]]
 199             then
 200                 echo "CertManager: Waiting for ${COUNT_CERTMANAGER_NOT_READY} of $((${COUNT_CERTMANAGER_NOT_READY}+${COUNT_CERTMANAGER_READY})) pods to be ready:"
 201                 echo "${CERTMANAGER_NOT_READY}"
 202                 echo
 203             fi
 204         fi
 205
 206         #------------ NEXT SAMPLE
 207         sleep ${sampling_period}
 208     done
 209
 210     ####################################################################################
 211     # OUTCOME
 212     ####################################################################################
 213     if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
 214     then
 215         echo
 216         FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN"
 217     else
 218         echo
 219         echo "K8S CLUSTER IS READY"
 220     fi
 221     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 222 }
 223
 224 # main
 225 while getopts ":D:d:i:-: " o; do
 226     case "${o}" in
 227         i)
 228             DEFAULT_IP="${OPTARG}"
 229             ;;
 230         d)
 231             OSM_CLUSTER_WORK_DIR="${OPTARG}"
 232             ;;
 233         D)
 234             OSM_DEVOPS="${OPTARG}"
 235             ;;
 236         -)
 237             [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
 238             [ "${OPTARG}" == "storageclass" ] && INSTALL_STORAGECLASS="y" && continue
 239             [ "${OPTARG}" == "metallb" ] && INSTALL_METALLB="y" && continue
 240             [ "${OPTARG}" == "nginx" ] && INSTALL_NGINX="y" && continue
 241             [ "${OPTARG}" == "certmgr" ] && INSTALL_CERTMANAGER="y" && continue
 242             [ "${OPTARG}" == "all" ] && INSTALL_STORAGECLASS="y" && INSTALL_METALLB="y" && INSTALL_NGINX="y" && INSTALL_CERTMANAGER="y" && continue
 243             echo -e "Invalid option: '--$OPTARG'\n" >&2
 244             exit 1
 245             ;;
 246         :)
 247             echo "Option -$OPTARG requires an argument" >&2
 248             exit 1
 249             ;;
 250         \?)
 251             echo -e "Invalid option: '-$OPTARG'\n" >&2
 252             exit 1
 253             ;;
 254         *)
 255             exit 1
 256             ;;
 257     esac
 258 done
 259
 260 source $OSM_DEVOPS/common/logging
 261 source $OSM_DEVOPS/common/track
 262
 263 echo "DEBUG_INSTALL=${DEBUG_INSTALL:-}"
 264 echo "DEFAULT_IP=${DEFAULT_IP:-}"
 265 echo "OSM_DEVOPS=${OSM_DEVOPS:-}"
 266 echo "OSM_CLUSTER_WORK_DIR=${OSM_CLUSTER_WORK_DIR:-}"
 267
 268 if [ -n "${INSTALL_STORAGECLASS}" ]; then
 269     install_k8s_storageclass
 270     track k8scluster k8s_storageclass_ok
 271 fi
 272 if [ -n "${INSTALL_METALLB}" ]; then
 273     install_helm_metallb
 274     track k8scluster k8s_metallb_ok
 275 fi
 276 if [ -n "${INSTALL_CERTMANAGER}" ]; then
 277     install_helm_certmanager
 278     track k8scluster k8s_certmanager_ok
 279 fi
 280 if [ -n "${INSTALL_NGINX}" ]; then
 281     install_helm_nginx
 282     track k8scluster k8s_nginx_ok
 283 fi
 284 check_for_readiness
 285 track k8scluster k8s_ready_ok
 286 if [ -n "${INSTALL_METALLB}" ]; then
 287     configure_ipaddresspool_metallb
 288 fi