installers/install_k3s_cluster.sh

   1 #!/bin/bash
   2 #
   3 #   Licensed under the Apache License, Version 2.0 (the "License");
   4 #   you may not use this file except in compliance with the License.
   5 #   You may obtain a copy of the License at
   6 #
   7 #       http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #   Unless required by applicable law or agreed to in writing, software
  10 #   distributed under the License is distributed on an "AS IS" BASIS,
  11 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #   See the License for the specific language governing permissions and
  13 #   limitations under the License.
  14 #
  15
  16 set +eux
  17
  18 # K3s releases: https://github.com/k3s-io/k3s/releases/
  19 K8S_CLIENT_VERSION="v1.29.3"
  20 K8S_VERSION="v1.29.3+k3s1"
  21
  22 # configure registry
  23 function configure_registry() {
  24     if [ -n "${DOCKER_PROXY_URL}" ]; then
  25         echo "Configuring docker proxy URL in /etc/rancher/k3s/registries.yaml"
  26         cat << EOF | sudo tee /etc/rancher/k3s/registries.yaml > /dev/null
  27 mirrors:
  28   docker.io:
  29     endpoint:
  30       - "${DOCKER_PROXY_URL}"
  31 EOF
  32     fi
  33     if [ -n "${DOCKER_REGISTRY_URL}" ]; then
  34         echo "Configuring docker private registry in /etc/rancher/k3s/registries.yaml"
  35         cat << EOF | sudo tee -a /etc/rancher/k3s/registries.yaml > /dev/null
  36 configs:
  37   ${DOCKER_REGISTRY_URL}:
  38     auth:
  39       username: ${DOCKER_REGISTRY_USER}
  40       password: ${DOCKER_REGISTRY_PASSWORD}
  41 EOF
  42     fi
  43 }
  44
  45 # installs k3s
  46 function install_k3s() {
  47     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  48     export INSTALL_K3S_EXEC="--disable traefik"
  49     curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K8S_VERSION} sh -s -
  50     sudo chmod 644 /etc/rancher/k3s/k3s.yaml
  51     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  52 }
  53
  54 # updates service nodeport range
  55 function update_service_nodeport_range() {
  56     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  57     sudo k3s server --kube-apiserver-arg=service-node-port-range=80-32767
  58     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  59 }
  60
  61 # checks cluster readiness
  62 function check_for_readiness() {
  63     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
  64     # Check for Ready node, takes ~30 seconds
  65     echo "Waiting for K8s nodes to be ready"
  66     local time_for_failure=60    # seconds broken
  67     local sampling_period=5     # seconds
  68     local counter=0
  69     local cluster_ready=""
  70     while (( counter < time_for_failure ))
  71     do
  72         kubectl get nodes |grep master |grep -v none | grep Ready
  73         if [ $? -eq 0 ] ; then
  74             echo "K8s cluster is ready"
  75             cluster_ready="y"
  76             break
  77         else
  78             echo "K8s cluster is not ready yet"
  79             counter=$((counter + sampling_period))
  80             sleep ${sampling_period}
  81         fi
  82     done
  83     [ -n "$cluster_ready" ] || FATAL_TRACK k8scluster "K3s cluster nodes not ready after $time_for_failure seconds."
  84
  85     echo "Waiting for pods to be ready"
  86     local time_for_readiness=20   # seconds ready
  87     local time_for_failure=100    # seconds broken
  88
  89     # Equivalent number of samples
  90     oks_threshold=$((time_for_readiness/${sampling_period}))     # No. ok samples to declare the system ready
  91     failures_threshold=$((time_for_failure/${sampling_period}))  # No. nok samples to declare the system broken
  92     failures_in_a_row=0
  93     oks_in_a_row=0
  94     ####################################################################################
  95     # Loop to check system readiness
  96     ####################################################################################
  97     K3S_NAMESPACE=kube-system
  98     while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
  99     do
 100         # State of pods rather than completed jobs
 101         K3S_PODS_STATE=$(kubectl get pod -n ${K3S_NAMESPACE} --no-headers |grep -v Completed 2>&1)
 102         K3S_PODS_READY=$(echo "${K3S_PODS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 103         K3S_PODS_NOT_READY=$(echo "${K3S_PODS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
 104         COUNT_K3S_PODS_READY=$(echo "${K3S_PODS_READY}"| grep -v -e '^$' | wc -l)
 105         COUNT_K3S_PODS_NOT_READY=$(echo "${K3S_PODS_NOT_READY}" | grep -v -e '^$' | wc -l)
 106
 107         # OK sample
 108         if [[ ${COUNT_K3S_PODS_NOT_READY} -eq 0 ]]
 109         then
 110             ((++oks_in_a_row))
 111             failures_in_a_row=0
 112             echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r
 113         # NOK sample
 114         else
 115             ((++failures_in_a_row))
 116             oks_in_a_row=0
 117             echo
 118             echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold}
 119
 120             # Reports failed pods in K3S
 121             if [[ "${COUNT_K3S_PODS_NOT_READY}" -ne 0 ]]
 122             then
 123                 echo "K3S kube-system: Waiting for ${COUNT_K3S_PODS_NOT_READY} of $((${COUNT_K3S_PODS_NOT_READY}+${COUNT_K3S_PODS_READY})) pods to be ready:"
 124                 echo "${K3S_PODS_NOT_READY}"
 125                 echo
 126             fi
 127         fi
 128
 129         #------------ NEXT SAMPLE
 130         sleep ${sampling_period}
 131     done
 132
 133     ####################################################################################
 134     # OUTCOME
 135     ####################################################################################
 136     if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
 137     then
 138         echo
 139         FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN"
 140     else
 141         echo
 142         echo "K8S CLUSTER IS READY"
 143     fi
 144     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 145 }
 146
 147 # Install kubectl client
 148 function install_kubectl() {
 149     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
 150     curl -LO "https://dl.k8s.io/release/${K8S_CLIENT_VERSION}/bin/linux/amd64/kubectl"
 151     sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
 152     rm kubectl
 153     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 154 }
 155
 156 # Initializes kubeconfig file
 157 function save_kubeconfig() {
 158     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
 159     KUBEDIR="${HOME}/.kube"
 160     KUBEFILE="$KUBEDIR/config"
 161     mkdir -p "${KUBEDIR}"
 162     K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
 163     sudo cp "${K3S_KUBECONFIG}" "${KUBEFILE}"
 164     sudo chown $(id -u):$(id -g) "${KUBEFILE}"
 165     sed -i "s#server: https://127.0.0.1#server: https://${DEFAULT_IP}#g" "${KUBEFILE}"
 166     chmod 700 "${KUBEFILE}"
 167     echo
 168     echo "Credentials saved at ${KUBEFILE}"
 169     echo
 170     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 171 }
 172
 173 # main
 174 while getopts ":D:i:p:d:u:P:-: " o; do
 175     case "${o}" in
 176         i)
 177             DEFAULT_IP="${OPTARG}"
 178             ;;
 179         D)
 180             OSM_DEVOPS="${OPTARG}"
 181             ;;
 182         p)
 183             DOCKER_PROXY_URL="${OPTARG}"
 184             ;;
 185         d)
 186             DOCKER_REGISTRY_URL="${OPTARG}"
 187             ;;
 188         u)
 189             DOCKER_REGISTRY_USER="${OPTARG}"
 190             ;;
 191         P)
 192             DOCKER_REGISTRY_PASSWORD="${OPTARG}"
 193             ;;
 194         -)
 195             [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
 196             echo -e "Invalid option: '--$OPTARG'\n" >&2
 197             exit 1
 198             ;;
 199         :)
 200             echo "Option -$OPTARG requires an argument" >&2
 201             exit 1
 202             ;;
 203         \?)
 204             echo -e "Invalid option: '-$OPTARG'\n" >&2
 205             exit 1
 206             ;;
 207         *)
 208             exit 1
 209             ;;
 210     esac
 211 done
 212
 213 DEBUG_INSTALL=${DEBUG_INSTALL:-}
 214 DEFAULT_IP=${DEFAULT_IP:-"127.0.0.1"}
 215 OSM_DEVOPS=${OSM_DEVOPS:-"/usr/share/osm-devops"}
 216 DOCKER_PROXY_URL=${DOCKER_PROXY_URL=-}
 217 DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL=-}
 218 DOCKER_REGISTRY_USER=${DOCKER_REGISTRY_USER=-}
 219 DOCKER_REGISTRY_PASSWORD=${DOCKER_REGISTRY_PASSWORD=-}
 220 echo "DEBUG_INSTALL=${DEBUG_INSTALL}"
 221 echo "DEFAULT_IP=${DEFAULT_IP}"
 222 echo "OSM_DEVOPS=${OSM_DEVOPS}"
 223 echo "DOCKER_PROXY_URL=${DOCKER_PROXY_URL}"
 224 echo "DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL}"
 225 echo "DOCKER_REGISTRY_USER=${DOCKER_REGISTRY_USER}"
 226 echo "HOME=$HOME"
 227
 228 source $OSM_DEVOPS/common/logging
 229 source $OSM_DEVOPS/common/track
 230
 231 configure_registry
 232 install_k3s
 233 track k8scluster k3s_install_ok
 234 check_for_readiness
 235 track k8scluster k3s_node_ready_ok
 236 # update_service_nodeport_range
 237 # check_for_readiness
 238 # track k8scluster k3s_update_nodeport_range_ok
 239 install_kubectl
 240 save_kubeconfig
 241 track k8scluster k3s_creds_ok