From: garciadeblas Date: Thu, 6 Jun 2024 12:26:24 +0000 (+0200) Subject: Feature 11041: Enable K3s as Kubernetes distro for OSM installation X-Git-Tag: release-v16.0-start~20 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=b37974111d2e1bc5d5b9916a800f682871e78290;p=osm%2Fdevops.git Feature 11041: Enable K3s as Kubernetes distro for OSM installation Change-Id: I42df97b07ae617ff22df32cc9eb511cf039eec82 Signed-off-by: garciadeblas --- diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 4747949e..679e834c 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -25,7 +25,7 @@ function usage(){ echo -e " -u : use specified repository url for osm packages" echo -e " -k : use specified repository public key url" echo -e " -a : use this apt proxy url when downloading apt packages (air-gapped installation)" - echo -e " -c : use a specific kubernetes engine (options: kubeadm, k3s, microk8s), default is kubeadm" + echo -e " -c : use a specific kubernetes engine (options: kubeadm, k3s), default is kubeadm" echo -e " -s namespace when installed using k8s, default is osm" echo -e " -H use specific juju host controller IP" echo -e " -S use VCA/juju secret key" @@ -420,14 +420,25 @@ function install_k8s_cluster() { KUBEADM_INSTALL_OPTS="-d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}" $OSM_DEVOPS/installers/install_kubeadm_cluster.sh ${KUBEADM_INSTALL_OPTS} || \ FATAL_TRACK k8scluster "install_kubeadm_cluster.sh failed" - K8SCLUSTER_ADDONS_INSTALL_OPTS="-i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}" + K8SCLUSTER_ADDONS_INSTALL_OPTS="-i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} --all" $OSM_DEVOPS/installers/install_cluster_addons.sh ${K8SCLUSTER_ADDONS_INSTALL_OPTS} || \ - FATAL_TRACK k8scluster "install_cluster_addons.sh failed" + FATAL_TRACK k8scluster "install_cluster_addons.sh failed for kubeadm cluster" + elif [ "${K8S_CLUSTER_ENGINE}" == "k3s" ]; then + K3S_INSTALL_OPTS="-D ${OSM_DEVOPS} ${DEBUG_INSTALL}" + # The K3s installation script will automatically take the HTTP_PROXY, HTTPS_PROXY and NO_PROXY, + # as well as the CONTAINERD_HTTP_PROXY, CONTAINERD_HTTPS_PROXY and CONTAINERD_NO_PROXY variables + # from the shell, if they are present, and write them to the environment file of k3s systemd service, + $OSM_DEVOPS/installers/install_k3s_cluster.sh ${K3S_INSTALL_OPTS} || \ + FATAL_TRACK k8scluster "install_k3s_cluster.sh failed" + K8SCLUSTER_ADDONS_INSTALL_OPTS="-i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} --certmgr --nginx" + $OSM_DEVOPS/installers/install_cluster_addons.sh ${K8SCLUSTER_ADDONS_INSTALL_OPTS} || \ + FATAL_TRACK k8scluster "install_cluster_addons.sh failed for k3s cluster" fi [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function deploy_osm() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function deploy_mongodb track deploy_osm deploy_mongodb_ok deploy_osm_helm_chart @@ -437,6 +448,7 @@ function deploy_osm() { install_osm_ngsa_service track deploy_osm install_osm_ngsa_ok fi + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function install_osm() { @@ -683,7 +695,7 @@ INSTALL_LXD="" SHOWOPTS="" ASSUME_YES="" APT_PROXY_URL="" -K8S_CLUSTER_ENGINE="kubeadm" +K8S_CLUSTER_ENGINE="k3s" DEBUG_INSTALL="" RELEASE="testing-daily" REPOSITORY="testing" @@ -754,7 +766,6 @@ while getopts ":a:c:r:n:k:u:R:D:o:O:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; do K8S_CLUSTER_ENGINE=${OPTARG} [ "${K8S_CLUSTER_ENGINE}" == "kubeadm" ] && continue [ "${K8S_CLUSTER_ENGINE}" == "k3s" ] && continue - [ "${K8S_CLUSTER_ENGINE}" == "microk8s" ] && continue echo -e "Invalid argument for -c : ' ${K8S_CLUSTER_ENGINE}'\n" >&2 usage && exit 1 ;; diff --git a/installers/install_cluster_addons.sh b/installers/install_cluster_addons.sh index 245ff266..58c64eec 100755 --- a/installers/install_cluster_addons.sh +++ b/installers/install_cluster_addons.sh @@ -15,6 +15,11 @@ set +eux +INSTALL_STORAGECLASS="" +INSTALL_METALLB="" +INSTALL_CERTMANAGER="" +INSTALL_NGINX="" + function install_k8s_storageclass() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function # Openebs versions can be found here: https://github.com/openebs/openebs/releases @@ -22,7 +27,7 @@ function install_k8s_storageclass() { echo "Installing OpenEBS" helm repo add openebs https://openebs.github.io/charts helm repo update - helm install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION} + helm upgrade --install --create-namespace --namespace openebs openebs openebs/openebs --version ${OPENEBS_VERSION} helm ls -n openebs local storageclass_timeout=400 local counter=0 @@ -53,13 +58,18 @@ function install_helm_metallb() { METALLB_VERSION="0.13.10" helm repo add metallb https://metallb.github.io/metallb helm repo update - helm install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION} + # kubectl create namespace metallb-system + # kubectl label namespaces metallb-system pod-security.kubernetes.io/enforce=privileged + # kubectl label namespaces metallb-system pod-security.kubernetes.io/audit=privileged + # kubectl label namespaces metallb-system pod-security.kubernetes.io/warn=privileged + helm upgrade --install --create-namespace --namespace metallb-system metallb metallb/metallb --version ${METALLB_VERSION} [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function configure_ipaddresspool_metallb() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function echo "Creating IP address pool manifest: ${OSM_CLUSTER_WORK_DIR}/metallb-ipaddrpool.yaml" + [ ! -d "$OSM_CLUSTER_WORK_DIR" ] && sudo mkdir -p $OSM_CLUSTER_WORK_DIR METALLB_IP_RANGE="$DEFAULT_IP/32" echo "apiVersion: metallb.io/v1beta1 kind: IPAddressPool @@ -81,7 +91,7 @@ function install_helm_certmanager() { CERTMANAGER_VERSION="v1.9.1" helm repo add jetstack https://charts.jetstack.io helm repo update - helm install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \ + helm upgrade --install cert-manager --create-namespace --namespace cert-manager jetstack/cert-manager \ --version ${CERTMANAGER_VERSION} --set installCRDs=true --set prometheus.enabled=false \ --set clusterResourceNamespace=osm \ --set extraArgs="{--enable-certificate-owner-ref=true}" @@ -116,7 +126,6 @@ function check_for_readiness() { OPENEBS_NAMESPACE=openebs METALLB_NAMESPACE=metallb-system CERTMANAGER_NAMESPACE=cert-manager - # STACK_NAME=osm # By default, "osm" # Equivalent number of samples oks_threshold=$((time_for_readiness/${sampling_period})) # No. ok samples to declare the system ready @@ -130,28 +139,34 @@ function check_for_readiness() { while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]] do # State of OpenEBS - OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1) - OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') - OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') - COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l) - COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l) + if [ -n "${INSTALL_STORAGECLASS}" ]; then + OPENEBS_STATE=$(kubectl get pod -n ${OPENEBS_NAMESPACE} --no-headers 2>&1) + OPENEBS_READY=$(echo "${OPENEBS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + OPENEBS_NOT_READY=$(echo "${OPENEBS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_OPENEBS_READY=$(echo "${OPENEBS_READY}"| grep -v -e '^$' | wc -l) + COUNT_OPENEBS_NOT_READY=$(echo "${OPENEBS_NOT_READY}" | grep -v -e '^$' | wc -l) + fi # State of MetalLB - METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1) - METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}') - METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}') - COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l) - COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l) + if [ -n "${INSTALL_METALLB}" ]; then + METALLB_STATE=$(kubectl get pod -n ${METALLB_NAMESPACE} --no-headers 2>&1) + METALLB_READY=$(echo "${METALLB_STATE}" | awk '$2=="1/1" || $2=="4/4" {printf ("%s\t%s\t\n", $1, $2)}') + METALLB_NOT_READY=$(echo "${METALLB_STATE}" | awk '$2!="1/1" && $2!="4/4" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_METALLB_READY=$(echo "${METALLB_READY}" | grep -v -e '^$' | wc -l) + COUNT_METALLB_NOT_READY=$(echo "${METALLB_NOT_READY}" | grep -v -e '^$' | wc -l) + fi # State of CertManager - CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1) - CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') - CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') - COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l) - COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l) + if [ -n "${INSTALL_CERTMANAGER}" ]; then + CERTMANAGER_STATE=$(kubectl get pod -n ${CERTMANAGER_NAMESPACE} --no-headers 2>&1) + CERTMANAGER_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_CERTMANAGER_READY=$(echo "${CERTMANAGER_READY}" | grep -v -e '^$' | wc -l) + COUNT_CERTMANAGER_NOT_READY=$(echo "${CERTMANAGER_NOT_READY}" | grep -v -e '^$' | wc -l) + fi # OK sample - if [[ $((${COUNT_OPENEBS_NOT_READY}+${COUNT_METALLB_NOT_READY})) -eq 0 ]] + if [[ $((${COUNT_OPENEBS_NOT_READY:-0}+${COUNT_METALLB_NOT_READY:-0}+${COUNT_CERTMANAGER_NOT_READY:-0})) -eq 0 ]] then ((++oks_in_a_row)) failures_in_a_row=0 @@ -164,7 +179,7 @@ function check_for_readiness() { echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold} # Reports failed pods in OpenEBS - if [[ "${COUNT_OPENEBS_NOT_READY}" -ne 0 ]] + if [[ "${COUNT_OPENEBS_NOT_READY:-0}" -ne 0 ]] then echo "OpenEBS: Waiting for ${COUNT_OPENEBS_NOT_READY} of $((${COUNT_OPENEBS_NOT_READY}+${COUNT_OPENEBS_READY})) pods to be ready:" echo "${OPENEBS_NOT_READY}" @@ -172,7 +187,7 @@ function check_for_readiness() { fi # Reports failed pods in MetalLB - if [[ "${COUNT_METALLB_NOT_READY}" -ne 0 ]] + if [[ "${COUNT_METALLB_NOT_READY:-0}" -ne 0 ]] then echo "MetalLB: Waiting for ${COUNT_METALLB_NOT_READY} of $((${COUNT_METALLB_NOT_READY}+${COUNT_METALLB_READY})) pods to be ready:" echo "${METALLB_NOT_READY}" @@ -180,7 +195,7 @@ function check_for_readiness() { fi # Reports failed pods in CertManager - if [[ "${COUNT_CERTMANAGER_NOT_READY}" -ne 0 ]] + if [[ "${COUNT_CERTMANAGER_NOT_READY:-0}" -ne 0 ]] then echo "CertManager: Waiting for ${COUNT_CERTMANAGER_NOT_READY} of $((${COUNT_CERTMANAGER_NOT_READY}+${COUNT_CERTMANAGER_READY})) pods to be ready:" echo "${CERTMANAGER_NOT_READY}" @@ -220,6 +235,11 @@ while getopts ":D:d:i:-: " o; do ;; -) [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue + [ "${OPTARG}" == "storageclass" ] && INSTALL_STORAGECLASS="y" && continue + [ "${OPTARG}" == "metallb" ] && INSTALL_METALLB="y" && continue + [ "${OPTARG}" == "nginx" ] && INSTALL_NGINX="y" && continue + [ "${OPTARG}" == "certmgr" ] && INSTALL_CERTMANAGER="y" && continue + [ "${OPTARG}" == "all" ] && INSTALL_STORAGECLASS="y" && INSTALL_METALLB="y" && INSTALL_NGINX="y" && INSTALL_CERTMANAGER="y" && continue echo -e "Invalid option: '--$OPTARG'\n" >&2 exit 1 ;; @@ -240,19 +260,29 @@ done source $OSM_DEVOPS/common/logging source $OSM_DEVOPS/common/track -echo "DEBUG_INSTALL=$DEBUG_INSTALL" -echo "DEFAULT_IP=$DEFAULT_IP" -echo "OSM_DEVOPS=$OSM_DEVOPS" -echo "OSM_CLUSTER_WORK_DIR=$OSM_CLUSTER_WORK_DIR" +echo "DEBUG_INSTALL=${DEBUG_INSTALL:-}" +echo "DEFAULT_IP=${DEFAULT_IP:-}" +echo "OSM_DEVOPS=${OSM_DEVOPS:-}" +echo "OSM_CLUSTER_WORK_DIR=${OSM_CLUSTER_WORK_DIR:-}" -install_k8s_storageclass -track k8scluster k8s_storageclass_ok -install_helm_metallb -track k8scluster k8s_metallb_ok -install_helm_certmanager -track k8scluster k8s_certmanager_ok -install_helm_nginx -track k8scluster k8s_nginx_ok +if [ -n "${INSTALL_STORAGECLASS}" ]; then + install_k8s_storageclass + track k8scluster k8s_storageclass_ok +fi +if [ -n "${INSTALL_METALLB}" ]; then + install_helm_metallb + track k8scluster k8s_metallb_ok +fi +if [ -n "${INSTALL_CERTMANAGER}" ]; then + install_helm_certmanager + track k8scluster k8s_certmanager_ok +fi +if [ -n "${INSTALL_NGINX}" ]; then + install_helm_nginx + track k8scluster k8s_nginx_ok +fi check_for_readiness track k8scluster k8s_ready_ok -configure_ipaddresspool_metallb +if [ -n "${INSTALL_METALLB}" ]; then + configure_ipaddresspool_metallb +fi diff --git a/installers/install_k3s_cluster.sh b/installers/install_k3s_cluster.sh new file mode 100755 index 00000000..aa5d6f75 --- /dev/null +++ b/installers/install_k3s_cluster.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +eux + +# K3s releases: https://github.com/k3s-io/k3s/releases/ +K8S_VERSION="v1.29.3+k3s1" + +# installs k3s +function install_k3s() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + export INSTALL_K3S_EXEC="--disable traefik" + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K8S_VERSION} sh -s - + sudo chmod 644 /etc/rancher/k3s/k3s.yaml + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# updates service nodeport range +function update_service_nodeport_range() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + sudo k3s server --kube-apiserver-arg=service-node-port-range=80-32767 + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# checks cluster readiness +function check_for_readiness() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + # Check for Ready node, takes ~30 seconds + echo "Waiting for K8s nodes to be ready" + local time_for_failure=60 # seconds broken + local sampling_period=5 # seconds + local counter=0 + local cluster_ready="" + while (( counter < time_for_failure )) + do + kubectl get nodes |grep master |grep -v none | grep Ready + if [ $? -eq 0 ] ; then + echo "K8s cluster is ready" + cluster_ready="y" + break + else + echo "K8s cluster is not ready yet" + counter=$((counter + sampling_period)) + sleep ${sampling_period} + fi + done + [ -n "$cluster_ready" ] || FATAL_TRACK k8scluster "K3s cluster nodes not ready after $time_for_failure seconds." + + echo "Waiting for pods to be ready" + local time_for_readiness=20 # seconds ready + local time_for_failure=100 # seconds broken + + # Equivalent number of samples + oks_threshold=$((time_for_readiness/${sampling_period})) # No. ok samples to declare the system ready + failures_threshold=$((time_for_failure/${sampling_period})) # No. nok samples to declare the system broken + failures_in_a_row=0 + oks_in_a_row=0 + #################################################################################### + # Loop to check system readiness + #################################################################################### + K3S_NAMESPACE=kube-system + while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]] + do + # State of pods rather than completed jobs + K3S_PODS_STATE=$(kubectl get pod -n ${K3S_NAMESPACE} --no-headers |grep -v Completed 2>&1) + K3S_PODS_READY=$(echo "${K3S_PODS_STATE}" | awk '$2=="1/1" || $2=="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + K3S_PODS_NOT_READY=$(echo "${K3S_PODS_STATE}" | awk '$2!="1/1" && $2!="2/2" {printf ("%s\t%s\t\n", $1, $2)}') + COUNT_K3S_PODS_READY=$(echo "${K3S_PODS_READY}"| grep -v -e '^$' | wc -l) + COUNT_K3S_PODS_NOT_READY=$(echo "${K3S_PODS_NOT_READY}" | grep -v -e '^$' | wc -l) + + # OK sample + if [[ ${COUNT_K3S_PODS_NOT_READY} -eq 0 ]] + then + ((++oks_in_a_row)) + failures_in_a_row=0 + echo -ne ===\> Successful checks: "${oks_in_a_row}"/${oks_threshold}\\r + # NOK sample + else + ((++failures_in_a_row)) + oks_in_a_row=0 + echo + echo Bootstraping... "${failures_in_a_row}" checks of ${failures_threshold} + + # Reports failed pods in K3S + if [[ "${COUNT_K3S_PODS_NOT_READY}" -ne 0 ]] + then + echo "K3S kube-system: Waiting for ${COUNT_K3S_PODS_NOT_READY} of $((${COUNT_K3S_PODS_NOT_READY}+${COUNT_K3S_PODS_READY})) pods to be ready:" + echo "${K3S_PODS_NOT_READY}" + echo + fi + fi + + #------------ NEXT SAMPLE + sleep ${sampling_period} + done + + #################################################################################### + # OUTCOME + #################################################################################### + if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]] + then + echo + FATAL_TRACK k8scluster "K8S CLUSTER IS BROKEN" + else + echo + echo "K8S CLUSTER IS READY" + fi + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# Retrieves and saves the credentials +function save_credentials() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + KUBEDIR="${HOME}/.kube" + KUBEFILE="$KUBEDIR/config" + mkdir -p "${KUBEDIR}" + sudo cp /etc/rancher/k3s/k3s.yaml "${KUBEFILE}" + sudo chown $(id -u):$(id -g) "${KUBEFILE}" + chmod 700 "${KUBEFILE}" + echo + echo "Credentials saved at ${KUBEFILE}" + echo + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# main +while getopts ":D:-: " o; do + case "${o}" in + D) + OSM_DEVOPS="${OPTARG}" + ;; + -) + [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue + echo -e "Invalid option: '--$OPTARG'\n" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument" >&2 + exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\n" >&2 + exit 1 + ;; + *) + exit 1 + ;; + esac +done + +source $OSM_DEVOPS/common/logging +source $OSM_DEVOPS/common/track + +echo "DEBUG_INSTALL=$DEBUG_INSTALL" +echo "OSM_DEVOPS=$OSM_DEVOPS" +echo "HOME=$HOME" + +install_k3s +track k8scluster k3s_install_ok +check_for_readiness +track k8scluster k3s_node_ready_ok +# update_service_nodeport_range +# check_for_readiness +# track k8scluster k3s_update_nodeport_range_ok +save_credentials +track k8scluster k3s_creds_ok