cloud-scripts/create-k8s.sh

   1 #!/usr/bin/env bash
   2 #######################################################################################
   3 # Copyright ETSI Contributors and Others.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #######################################################################################
  18
  19 # Create a new VM for installing a k8s cluster and its NSG on Azure. SSH key pair ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub must exist.
  20 # To do this it reads the following environment variables:
  21 # - K8S_IMAGE_NAME: name of the new VM
  22 # - RESOURCE_GROUP: name of the resource-group where the VM will be created
  23 # - VNET_NAME: name of the virtual network when creating a new one or referencing an existing one
  24 # - VIM_MGMT_NET: name or ID of the subnet to which the VM will be connected
  25 # - SOURCE_IMAGE_NAME: name of operating system image used (e.g. "Canonical:0001-com-ubuntu-server-jammy:22_04-lts:latest")
  26 # - K8S_FLAVOR_NAME: the VM size to be created (e.g. "Standard_A2_v2")
  27 # - PRIORITY: "Low", "Regular" or Spot"
  28 function create_azure_vm {
  29     set -eux
  30     az vm create --resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --image "${SOURCE_IMAGE_NAME}" --size "${K8S_FLAVOR_NAME}" --vnet-name "${VNET_NAME}" --subnet "${VIM_MGMT_NET}" --public-ip-address "" --admin-username ubuntu --priority "${PRIORITY}"
  31     export K8S_IP=$(az vm show -d -g "${RESOURCE_GROUP}" -n "${K8S_IMAGE_NAME}" --query privateIps | tr -d \")
  32     # Add a security group rule
  33     INTERFACE_ID=$(az vm show --resource-group ${RESOURCE_GROUP} --name ${K8S_IMAGE_NAME} --query networkProfile.networkInterfaces[0].id)
  34     INTERFACE_ID=${INTERFACE_ID:1:-1}
  35     SECURITY_GROUP_ID=$(az network nic show --id ${INTERFACE_ID} --query networkSecurityGroup.id)
  36     SECURITY_GROUP_ID=${SECURITY_GROUP_ID:1:-1}
  37     SECURITY_GROUP_NAME=$(az resource show --ids ${SECURITY_GROUP_ID} --query name)
  38     SECURITY_GROUP_NAME=${SECURITY_GROUP_NAME:1:-1}
  39     az network nsg rule create -n microk8s --nsg-name ${SECURITY_GROUP_NAME} --priority 2000 -g ${RESOURCE_GROUP}  --description "Microk8s port" --protocol TCP --destination-port-ranges 16443
  40 }
  41
  42 # Create a new VM for installing a k8s cluster on GCP. SSH key pair ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub must exist.
  43 # To do this it reads the following environment variables:
  44 # - K8S_IMAGE_NAME: name of the new VM
  45 # - GCP_PROJECT: name of project where the VM will be allocated
  46 # - GCP_ZONE: name of the zone (e.g. "europe-west1-b")
  47 # - VIM_MGMT_NET: name or ID of the subnet to which the VM will be connected
  48 # - GCP_MACHINE_TYPE: machine type used for the instance (e.g. "e2-standard-4", run 'gcloud compute machine-types list')
  49 # - GCP_IMAGE_PROJECT: the Google Cloud project against which all image and image family references will be resolved (e.g. "ubuntu-os-cloud")
  50 # - GCP_IMAGE_FAMILY: the image family for the operating system that the boot disk will be initialized with (e.g. "ubuntu-2204-lts")
  51 # - GCP_DISK_SIZE: disk size ib GB
  52 function create_gcp_vm {
  53     gcloud compute instances create "${K8S_IMAGE_NAME}" --project="${GCP_PROJECT}" --zone="${GCP_ZONE}" --machine-type="${GCP_MACHINE_TYPE}" \
  54     --network-interface=network-tier=PREMIUM,subnet=${VIM_MGMT_NET} --maintenance-policy=MIGRATE \
  55     --image-family=${GCP_IMAGE_FAMILY} --image-project=${GCP_IMAGE_PROJECT} --metadata-from-file=ssh-keys=${HOME}/.ssh/id_rsa.pub \
  56     --create-disk=auto-delete=yes,boot=yes,image-family=${GCP_IMAGE_FAMILY},image-project=${GCP_IMAGE_PROJECT},device-name=${K8S_IMAGE_NAME},mode=rw,size=${GCP_DISK_SIZE} -q
  57 }
  58
  59 # Install via SSH a microk8s cluster on a VM. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
  60 # To do this it reads the following environment variables:
  61 # - K8S_IP: IP address of the target machine
  62 function install_remote_microk8s {
  63     set +e
  64     ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@"${K8S_IP}" 'sudo apt-get update -y && sudo apt-get upgrade -y && sudo reboot'
  65     sleep 90
  66     ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${K8S_IP} << EOF 2>&1
  67 set -eux
  68 sudo snap install yq
  69 sudo snap install microk8s --classic
  70 sudo usermod -a -G microk8s ubuntu
  71 newgrp microk8s
  72 sudo microk8s.status --wait-ready
  73 sudo microk8s.enable storage dns
  74 set +eux
  75 EOF
  76
  77     # Enable MetalLB
  78     ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${K8S_IP} << 'EOF' 2>&1
  79 set -eux
  80 PRIVATE_IP=$(hostname -I | awk '{print $1}')
  81 echo ${PRIVATE_IP}
  82 sudo microk8s.enable metallb:${PRIVATE_IP}-${PRIVATE_IP}
  83 EOF
  84
  85     # Update the certificate to allow connections from outside as well
  86     ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${K8S_IP} << EOF 2>&1
  87 set -aux
  88 sudo sed -i "s/\#MOREIPS/IP.3 = ${K8S_IP}/g" /var/snap/microk8s/current/certs/csr.conf.template
  89 cat /var/snap/microk8s/current/certs/csr.conf.template
  90 EOF
  91
  92     # Save the credentials
  93     echo ================================================================
  94     echo K8s cluster credentials:
  95     echo ================================================================
  96     echo
  97     ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${K8S_IP} \
  98 'sudo microk8s.config' | sed  "s/server: .*/server: https:\/\/${K8S_IP}:16443/g" \
  99 | tee ${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
 100 }
 101
 102 # Create an AKS cluster with one node on Azure using a new subnet. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
 103 # Required the following environment variables:
 104 # - K8S_IMAGE_NAME: name of the AKS cluster to be created
 105 # - RESOURCE_GROUP: name of the resource-group where the VM will be created
 106 # - VNET_NAME: name of the virtual network when creating a new one or referencing an existing one
 107 # - K8S_FLAVOR_NAME: the VM size to be created (e.g. "Standard_D4as_v4")
 108 # IMPORTANT: required a vnet at least /16 because the it's created a new subnet with a random third byte
 109 function create_azure_aks {
 110     # Gets first subnet prefix and creates a new one with random number in third byte
 111     set -eux
 112     SUBNET_PREFIX=$(az network vnet show --resource-group "${RESOURCE_GROUP}" --name "${VNET_NAME}" --query subnets[0].addressPrefix -o tsv)
 113     IFS=. read BYTE1 BYTE2 BYTE3 BYTE4 <<< "$SUBNET_PREFIX"
 114     IFS=/ read BYTE4 MASK <<< "$BYTE4"
 115     if [ "$MASK" -ge "24" ]; then
 116         BYTE3=$((100 + ($RANDOM % 50) * 2))
 117         PREFIX="$BYTE1.$BYTE2.${BYTE3}.$BYTE4/$MASK"
 118         SUBNET_NAME="aks-${BYTE3}"
 119         BYTE3=$((BYTE3 + 1))
 120         CIDR="$BYTE1.$BYTE2.${BYTE3}.$BYTE4/$MASK"
 121         DNS_IP="$BYTE1.$BYTE2.${BYTE3}.10"
 122         # Verifies that new subnet does not exist previously
 123         az network vnet subnet show --resource-group "${RESOURCE_GROUP}" --vnet-name "${VNET_NAME}" --name "$SUBNET_NAME" -o table
 124         if [ "$?" -ne 0 ]; then
 125             # Creates the subnet in $VNET_NAME network
 126             az network vnet subnet create --resource-group "${RESOURCE_GROUP}" --vnet-name "${VNET_NAME}" --name "$SUBNET_NAME" --address-prefixes "${PREFIX}"
 127             if [ "$?" -eq 0 ]; then
 128                 SUBNET_ID=$(az network vnet subnet show --resource-group OSM-CTIO --vnet-name "${VNET_NAME}" --name "${SUBNET_NAME}" --query id -o tsv)
 129
 130                 # Creates k8s cluster
 131                 az aks create -y --resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --node-count 1 --node-vm-size "${K8S_FLAVOR_NAME}" --dns-service-ip "${DNS_IP}" --network-plugin kubenet --service-cidr "${CIDR}" --vnet-subnet-id "${SUBNET_ID}"
 132                 az aks get-credentials --resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --admin -f ${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
 133             fi
 134         fi
 135     fi
 136 }
 137
 138 # Create a GKE cluster with one node on GCP. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
 139 # Required the following environment variables:
 140 # - K8S_IMAGE_NAME: name of the GKE cluster to be created
 141 # - GCP_PROJECT: name of project where the cluster will be allocated
 142 # - GCP_ZONE: name of the zone (e.g. "europe-west1-b")
 143 # - GCP_MACHINE_TYPE: machine type used for the instance (e.g. "e2-standard-4", run 'gcloud compute machine-types list')
 144 # - GCP_DISK_SIZE: disk size ib GB
 145 function create_gcp_gke {
 146     gcloud container clusters create "${K8S_IMAGE_NAME}" --project="${GCP_PROJECT}" --zone="${GCP_ZONE}" --num-nodes=1 --machine-type="${GCP_MACHINE_TYPE}" --disk-size "${GCP_DISK_SIZE}" --enable-ip-alias --image-type "COS_CONTAINERD"
 147 }
 148
 149 # Get kubeconfig.yaml from a GKE cluster on GCP. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
 150 function get_gke_kubeconfig {
 151     set -eu -o pipefail
 152     FILE=${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
 153     SA=osm-sa
 154     NAMESPACE=osm
 155     echo "Creating the Kubernetes Service Account with minimal RBAC permissions."
 156     kubectl apply -f - <<EOF
 157 apiVersion: v1
 158 kind: Namespace
 159 metadata:
 160   name: ${NAMESPACE}
 161 ---
 162 apiVersion: v1
 163 kind: ServiceAccount
 164 metadata:
 165   name: ${SA}
 166   namespace: ${NAMESPACE}
 167 ---
 168 apiVersion: rbac.authorization.k8s.io/v1
 169 kind: ClusterRole
 170 metadata:
 171   name: osm-role
 172 rules:
 173 - apiGroups:
 174   - ""
 175   resources:
 176   - users
 177   - groups
 178   - serviceaccounts
 179   verbs:
 180   - impersonate
 181 - apiGroups:
 182   - ""
 183   resources:
 184   - pods
 185   verbs:
 186   - get
 187 - apiGroups:
 188   - "authorization.k8s.io"
 189   resources:
 190   - selfsubjectaccessreviews
 191   - selfsubjectrulesreviews
 192   verbs:
 193   - create
 194 ---
 195 apiVersion: rbac.authorization.k8s.io/v1
 196 kind: ClusterRoleBinding
 197 metadata:
 198   name: osm-crb
 199 roleRef:
 200   apiGroup: rbac.authorization.k8s.io
 201   kind: ClusterRole
 202   name: cluster-admin
 203 subjects:
 204 - kind: ServiceAccount
 205   name: ${SA}
 206   namespace: ${NAMESPACE}
 207 EOF
 208
 209     # Get the service account token and CA cert.
 210     SA_SECRET_NAME=$(kubectl get -n ${NAMESPACE} sa/${SA} -o "jsonpath={.secrets[0]..name}")
 211     # Note: service account token is stored base64-encoded in the secret but must
 212     # be plaintext in kubeconfig.
 213     SA_TOKEN=$(kubectl get -n ${NAMESPACE} secrets/${SA_SECRET_NAME} -o "jsonpath={.data['token']}" | base64 ${BASE64_DECODE_FLAG})
 214     CA_CERT=$(kubectl get -n ${NAMESPACE} secrets/${SA_SECRET_NAME} -o "jsonpath={.data['ca\.crt']}")
 215
 216     # Extract cluster IP from the current context
 217     CURRENT_CONTEXT=$(kubectl config current-context)
 218     CURRENT_CLUSTER=$(kubectl config view -o jsonpath="{.contexts[?(@.name == \"${CURRENT_CONTEXT}\"})].context.cluster}")
 219     CURRENT_CLUSTER_ADDR=$(kubectl config view -o jsonpath="{.clusters[?(@.name == \"${CURRENT_CLUSTER}\"})].cluster.server}")
 220
 221     echo "Writing $FILE"
 222     cat > $FILE <<EOF
 223 apiVersion: v1
 224 clusters:
 225 - cluster:
 226     certificate-authority-data: ${CA_CERT}
 227     server: ${CURRENT_CLUSTER_ADDR}
 228   name: ${CURRENT_CLUSTER}
 229 contexts:
 230 - context:
 231     cluster: ${CURRENT_CLUSTER}
 232     user: ${CURRENT_CLUSTER}-${SA}
 233   name: ${CURRENT_CONTEXT}
 234 current-context: ${CURRENT_CONTEXT}
 235 kind: Config
 236 preferences: {}
 237 users:
 238 - name: ${CURRENT_CLUSTER}-${SA}
 239   user:
 240     token: ${SA_TOKEN}
 241 EOF
 242
 243     echo "Done!"
 244 }
 245
 246 # Name of the new VM for K8s (adds a timestamp)
 247 export K8S_IMAGE_NAME=k8stest$(date '+%Y%m%d%H%M')
 248
 249 # Default USE_PAAS_K8S is "FALSE"
 250 if [ -z "${USE_PAAS_K8S}" ]; then
 251     USE_PAAS_K8S="FALSE"
 252 fi
 253
 254 # Log new environment variables
 255 mkdir -p ${ROBOT_REPORT_FOLDER}
 256 cat <<EOF > ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 257 export CLOUD_TYPE="${CLOUD_TYPE}"
 258 export USE_PAAS_K8S="${USE_PAAS_K8S}"
 259 EOF
 260
 261 # Branch by USE_PAAS_K8S and CLOUD_TYPE values
 262 if [ "${USE_PAAS_K8S}" == "FALSE" ]; then
 263     echo "Creating a new IaaS k8s cluster in ${CLOUD_TYPE}"
 264     if [ "${CLOUD_TYPE}" == "azure" ]; then
 265         # Create VM on Azure
 266         create_azure_vm
 267     elif [ "${CLOUD_TYPE}" == "gcp" ]; then
 268         # Create VM on GCP
 269         create_gcp_vm
 270     else
 271         echo "CLOUD_TYPE '${CLOUD_TYPE}' not valid for IaaS"
 272         exit
 273     fi
 274     # Add environment variables
 275     echo "export K8S_IP=\"${K8S_IP}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 276     echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 277
 278     # MicroK8s installation
 279     install_remote_microk8s
 280 else
 281     echo "Creating a new PaaS k8s cluster in ${CLOUD_TYPE}"
 282     if [ "${CLOUD_TYPE}" == "azure" ]; then
 283         create_azure_aks
 284         echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 285     elif [ "${CLOUD_TYPE}" == "gcp" ]; then
 286         create_gcp_gke
 287         get_gke_kubeconfig
 288         echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 289     else
 290         echo "CLOUD_TYPE '${CLOUD_TYPE}' not valid for PaaS"
 291     fi
 292 fi
 293
 294 # Add K8S_CREDENTIALS environment variable
 295 echo "export K8S_CREDENTIALS=${ROBOT_REPORT_FOLDER}/kubeconfig.yaml" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 296 echo File with new environment was created at ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
 297