Newer
Older
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set +eux
K8S_VERSION=1.28.2-00
# installs kubernetes packages
function install_kube() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
# Kubernetes releases can be found here: https://kubernetes.io/releases/
# To check other available versions, run the following command
# curl -s https://packages.cloud.google.com/apt/dists/kubernetes-xenial/main/binary-amd64/Packages | grep Version | awk '{print $2}'
sudo apt-get -y update && sudo apt-get install -y apt-transport-https ca-certificates curl
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get -y update
echo "Installing Kubernetes Packages ..."
sudo apt-get install -y kubelet=${K8S_VERSION} kubeadm=${K8S_VERSION} kubectl=${K8S_VERSION}
sudo apt-mark hold kubelet kubeadm kubectl
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# check and track kube packages installation
function check_and_track_kube_install() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
kubelet_version=$(dpkg -s kubelet|grep Version|awk '{print $2}')
[ -n "${kubelet_version}" ] || FATAL_TRACK k8scluster "Kubelet was not installed."
kubeadm_version=$(dpkg -s kubeadm|grep Version|awk '{print $2}')
[ -n "${kubeadm_version}" ] || FATAL_TRACK k8scluster "Kubeadm was not installed."
kubectl_version=$(dpkg -s kubectl|grep Version|awk '{print $2}')
[ -n "${kubectl_version}" ] || FATAL_TRACK k8scluster "Kubectl was not installed."
track k8scluster install_k8s_ok none none none kubelet ${kubelet_version} none none kubeadm ${kubeadm_version} none none kubectl ${kubectl_version} none none
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# initializes kubernetes control plane
function init_kubeadm() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
sudo swapoff -a
sudo sed -i.bak '/.*none.*swap/s/^\(.*\)$/#\1/g' /etc/fstab
sudo kubeadm init --config $1 --dry-run || FATAL_TRACK k8scluster "kubeadm init dry-run failed"
sudo kubeadm init --config $1
sleep 5
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# Initializes kubeconfig file
function kube_config_dir() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
K8S_MANIFEST_DIR="/etc/kubernetes/manifests"
[ ! -d $K8S_MANIFEST_DIR ] && FATAL_TRACK k8scluster "Kubernetes folder $K8S_MANIFEST_DIR was not found"
mkdir -p $HOME/.kube
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# test kubernetes installation
function check_and_track_init_k8s() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
echo "Reading existing namespaces"
kubectl get ns || FATAL_TRACK k8scluster "Failed getting namespaces"
track k8scluster init_k8s_ok
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# deploys flannel as daemonsets
function deploy_cni_provider() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
CNI_DIR="$(mktemp -d -q --tmpdir "flannel.XXXXXX")"
trap 'rm -rf "${CNI_DIR}"' EXIT
KUBE_FLANNEL_FILE_URL="https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"
wget --retry-on-host-error --retry-on-http-error 404,429,503 --tries=5 "${KUBE_FLANNEL_FILE_URL}" -P $CNI_DIR
[ ! -f $CNI_DIR/kube-flannel.yml ] && FATAL_TRACK k8scluster "Cannot Install Flannel because $CNI_DIR/kube-flannel.yml was not found. Maybe the file ${KUBE_FLANNEL_FILE_URL} is temporarily not accessible"
kubectl apply -f $CNI_DIR
[ $? -ne 0 ] && FATAL_TRACK k8scluster "Cannot Install Flannel"
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# taints K8s master node
function taint_master_node() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
K8S_MASTER=$(kubectl get nodes | awk '$3~/control-plane/'| awk '{print $1; exit}')
kubectl taint node $K8S_MASTER node-role.kubernetes.io/control-plane:NoSchedule-
sleep 5
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# check and track kube packages installation
function check_and_track_k8s_ready_before_helm() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
kubectl get events || FATAL_TRACK k8scluster "Failed getting events"
track k8scluster k8s_ready_before_helm
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
#Helm releases can be found here: https://github.com/helm/helm/releases
function install_helm() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
if ! [[ "$(helm version --short 2>/dev/null)" =~ ^v3.* ]]; then
# Helm is not installed. Install helm
echo "Helm3 is not installed, installing ..."
curl https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz --output helm-${HELM_VERSION}.tar.gz
tar -zxvf helm-${HELM_VERSION}.tar.gz
sudo mv linux-amd64/helm /usr/local/bin/helm
rm -r linux-amd64
rm helm-${HELM_VERSION}.tar.gz
else
echo "Helm3 is already installed. Skipping installation..."
helm version || FATAL_TRACK k8scluster "Could not obtain helm version. Maybe helm client was not installed"
helm repo add stable https://charts.helm.sh/stable || FATAL_TRACK k8scluster "Helm repo stable could not be added"
helm repo update || FATAL_TRACK k8scluster "Helm repo stable could not be updated"
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
function install_k8s_storageclass() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
# Openebs versions can be found here: https://github.com/openebs/openebs/releases
OPENEBS_VERSION="3.7.0"
echo "Installing OpenEBS"
helm repo add openebs https://openebs.github.io/charts
helm repo update
helm install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION}
helm ls -n openebs
local storageclass_timeout=400
local counter=0
local storageclass_ready=""
echo "Waiting for storageclass"
while (( counter < storageclass_timeout ))
do
kubectl get storageclass openebs-hostpath &> /dev/null
if [ $? -eq 0 ] ; then
echo "Storageclass available"
storageclass_ready="y"
break
else
counter=$((counter + 15))
sleep 15
fi
done
[ -n "$storageclass_ready" ] || FATAL_TRACK k8scluster "Storageclass not ready after $storageclass_timeout seconds. Cannot install openebs"
kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
#installs metallb from helm
function install_helm_metallb() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
echo "Installing MetalLB"
helm repo add metallb https://metallb.github.io/metallb
helm repo update
helm install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION}
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
function configure_ipaddresspool_metallb() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
echo "Creating IP address pool manifest: ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
METALLB_IP_RANGE="$DEFAULT_IP/32"
echo "apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: first-pool
namespace: metallb-system
spec:
addresses:
- ${METALLB_IP_RANGE}" | sudo tee -a ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml
echo "Applying IP address pool manifest: kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml"
kubectl apply -f ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml || FATAL_TRACK k8scluster "Cannot create IP address Pool in MetalLB"
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
#installs cert-manager
function install_helm_certmanager() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
echo "Installing cert-manager"
CERTMANAGER_VERSION="v1.9.1"
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \
--version ${CERTMANAGER_VERSION} --set installCRDs=true --set prometheus.enabled=false \
--set clusterResourceNamespace=osm \
--set extraArgs="{--enable-certificate-owner-ref=true}"
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#checks openebs and metallb readiness
function check_for_readiness() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
# Default input values
sampling_period=2 # seconds
time_for_readiness=20 # seconds ready
time_for_failure=200 # seconds broken
OPENEBS_NAMESPACE=openebs
METALLB_NAMESPACE=metallb-system
# STACK_NAME=osm # By default, "osm"
# Equivalent number of samples
oks_threshold=$((time_for_readiness/${sampling_period})) # No. ok samples to declare the system ready
failures_threshold=$((time_for_failure/${sampling_period})) # No. nok samples to declare the system broken
failures_in_a_row=0
oks_in_a_row=0
####################################################################################
# Loop to check system readiness
####################################################################################
while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
do
# State of OpenEBS
OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1)
OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l)
COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l)
# State of MetalLB
METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1)
METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}')
COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l)
COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l)
# State of CertManager
CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1)
CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}')
COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l)
COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l)
# OK sample
if [[ $((${COUNT_OPENEBS_NOT_READY}+${COUNT_METALLB_NOT_READY})) -eq 0 ]]
then
((++oks_in_a_row))
failures_in_a_row=0
echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r
# NOK sample
else
((++failures_in_a_row))
oks_in_a_row=0
echo
echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold}
# Reports failed pods in OpenEBS
if [[ "${COUNT_OPENEBS_NOT_READY}" -ne 0 ]]
then
echo "OpenEBS: Waiting for ${COUNT_OPENEBS_NOT_READY} of $((${COUNT_OPENEBS_NOT_READY}+${COUNT_OPENEBS_READY})) pods to be ready:"
echo "${OPENEBS_NOT_READY}"
echo
fi
# Reports failed pods in MetalLB
if [[ "${COUNT_METALLB_NOT_READY}" -ne 0 ]]
then
echo "MetalLB: Waiting for ${COUNT_METALLB_NOT_READY} of $((${COUNT_METALLB_NOT_READY}+${COUNT_METALLB_READY})) pods to be ready:"
echo "${METALLB_NOT_READY}"
echo
fi
# Reports failed pods in CertManager
if [[ "${COUNT_CERTMANAGER_NOT_READY}" -ne 0 ]]
then
echo "CertManager: Waiting for ${COUNT_CERTMANAGER_NOT_READY} of $((${COUNT_CERTMANAGER_NOT_READY}+${COUNT_CERTMANAGER_READY})) pods to be ready:"
echo "${CERTMANAGER_NOT_READY}"
echo
fi
fi
#------------ NEXT SAMPLE
sleep ${sampling_period}
done
####################################################################################
# OUTCOME
####################################################################################
if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
then
echo
FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN"
else
echo
echo "K8S CLUSTER IS READY"
fi
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
#removes osm deployments and services
function remove_k8s_namespace() {
[ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
echo "Deleting existing namespace $1: kubectl delete ns $1"
kubectl delete ns $1 2>/dev/null
[ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
}
# main
while getopts ":D:d:i:-: " o; do
case "${o}" in
i)
DEFAULT_IP="${OPTARG}"
;;
d)
OSM_CLUSTER_WORK_DIR="${OPTARG}"
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
;;
D)
OSM_DEVOPS="${OPTARG}"
;;
-)
[ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
echo -e "Invalid option: '--$OPTARG'\n" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument" >&2
exit 1
;;
\?)
echo -e "Invalid option: '-$OPTARG'\n" >&2
exit 1
;;
*)
exit 1
;;
esac
done
source $OSM_DEVOPS/common/logging
source $OSM_DEVOPS/common/track
echo "DEBUG_INSTALL=$DEBUG_INSTALL"
echo "DEFAULT_IP=$DEFAULT_IP"
echo "OSM_DEVOPS=$OSM_DEVOPS"
echo "OSM_CLUSTER_WORK_DIR=$OSM_CLUSTER_WORK_DIR"
echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR"
echo "HOME=$HOME"
garciadeblas
committed
echo "Creating folders for installation"
[ ! -d "$OSM_CLUSTER_WORK_DIR" ] && sudo mkdir -p $OSM_CLUSTER_WORK_DIR
echo "Copying kubeadm-config from $OSM_DEVOPS/installers/kubeadm-config.yaml to $OSM_CLUSTER_WORK_DIR/kubeadm-config.yaml"
sudo cp -b $OSM_DEVOPS/installers/kubeadm-config.yaml $OSM_CLUSTER_WORK_DIR/kubeadm-config.yaml
check_and_track_kube_install
init_kubeadm $OSM_CLUSTER_WORK_DIR/kubeadm-config.yaml
check_and_track_init_k8s
deploy_cni_provider
taint_master_node
check_and_track_k8s_ready_before_helm
remove_k8s_namespace osm
track k8scluster install_helm_ok
install_k8s_storageclass
track k8scluster k8s_storageclass_ok
track k8scluster k8s_metallb_ok
install_helm_certmanager
track k8scluster k8s_certmanager_ok
track k8scluster k8s_ready_ok
configure_ipaddresspool_metallb