2 #######################################################################################
3 # Copyright ETSI Contributors and Others.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #######################################################################################
19 # Create a new VM for installing a k8s cluster and its NSG on Azure. SSH key pair ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub must exist.
20 # To do this it reads the following environment variables:
21 # - K8S_IMAGE_NAME: name of the new VM
22 # - RESOURCE_GROUP: name of the resource-group where the VM will be created
23 # - VNET_NAME: name of the virtual network when creating a new one or referencing an existing one
24 # - VIM_MGMT_NET: name or ID of the subnet to which the VM will be connected
25 # - SOURCE_IMAGE_NAME: name of operating system image used (e.g. "Canonical:0001-com-ubuntu-server-jammy:22_04-lts:latest")
26 # - K8S_FLAVOR_NAME: the VM size to be created (e.g. "Standard_A2_v2")
27 # - PRIORITY: "Low", "Regular" or Spot"
28 function create_azure_vm
{
30 az vm create
--resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --image "${SOURCE_IMAGE_NAME}" --size "${K8S_FLAVOR_NAME}" --vnet-name "${VNET_NAME}" --subnet "${VIM_MGMT_NET}" --public-ip-address "" --admin-username ubuntu --priority "${PRIORITY}"
31 export K8S_IP
=$
(az vm show
-d -g "${RESOURCE_GROUP}" -n "${K8S_IMAGE_NAME}" --query privateIps |
tr -d \")
32 # Add a security group rule
33 INTERFACE_ID
=$
(az vm show
--resource-group ${RESOURCE_GROUP} --name ${K8S_IMAGE_NAME} --query networkProfile.networkInterfaces
[0].id
)
34 INTERFACE_ID
=${INTERFACE_ID:1:-1}
35 SECURITY_GROUP_ID
=$
(az network nic show
--id ${INTERFACE_ID} --query networkSecurityGroup.id
)
36 SECURITY_GROUP_ID
=${SECURITY_GROUP_ID:1:-1}
37 SECURITY_GROUP_NAME
=$
(az resource show
--ids ${SECURITY_GROUP_ID} --query name
)
38 SECURITY_GROUP_NAME
=${SECURITY_GROUP_NAME:1:-1}
39 az network nsg rule create
-n microk8s
--nsg-name ${SECURITY_GROUP_NAME} --priority 2000 -g ${RESOURCE_GROUP} --description "Microk8s port" --protocol TCP
--destination-port-ranges 16443
42 # Create a new VM for installing a k8s cluster on GCP. SSH key pair ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub must exist.
43 # To do this it reads the following environment variables:
44 # - K8S_IMAGE_NAME: name of the new VM
45 # - GCP_PROJECT: name of project where the VM will be allocated
46 # - GCP_ZONE: name of the zone (e.g. "europe-west1-b")
47 # - VIM_MGMT_NET: name or ID of the subnet to which the VM will be connected
48 # - GCP_MACHINE_TYPE: machine type used for the instance (e.g. "e2-standard-4", run 'gcloud compute machine-types list')
49 # - GCP_IMAGE_PROJECT: the Google Cloud project against which all image and image family references will be resolved (e.g. "ubuntu-os-cloud")
50 # - GCP_IMAGE_FAMILY: the image family for the operating system that the boot disk will be initialized with (e.g. "ubuntu-2204-lts")
51 # - GCP_DISK_SIZE: disk size ib GB
52 function create_gcp_vm
{
53 gcloud compute instances create
"${K8S_IMAGE_NAME}" --project="${GCP_PROJECT}" --zone="${GCP_ZONE}" --machine-type="${GCP_MACHINE_TYPE}" \
54 --network-interface=network-tier
=PREMIUM
,subnet
=${VIM_MGMT_NET} --maintenance-policy=MIGRATE \
55 --image-family=${GCP_IMAGE_FAMILY} --image-project=${GCP_IMAGE_PROJECT} --metadata-from-file=ssh-keys=${HOME}/.ssh
/id_rsa.pub \
56 --create-disk=auto-delete
=yes,boot
=yes,image-family
=${GCP_IMAGE_FAMILY},image-project=${GCP_IMAGE_PROJECT},device-name=${K8S_IMAGE_NAME},mode=rw,size=${GCP_DISK_SIZE} -q
59 # Install via SSH a microk8s cluster on a VM. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
60 # To do this it reads the following environment variables:
61 # - K8S_IP: IP address of the target machine
62 function install_remote_microk8s
{
64 ssh -T -o StrictHostKeyChecking
=no
-o UserKnownHostsFile
=/dev
/null ubuntu@
"${K8S_IP}" 'sudo apt-get update -y && sudo apt-get upgrade -y && sudo reboot'
66 ssh -T -o StrictHostKeyChecking
=no
-o UserKnownHostsFile
=/dev
/null ubuntu@
${K8S_IP} << EOF 2>&1
69 sudo snap install microk8s --classic
70 sudo usermod -a -G microk8s ubuntu
72 sudo microk8s.status --wait-ready
73 sudo microk8s.enable storage dns
78 ssh -T -o StrictHostKeyChecking
=no
-o UserKnownHostsFile
=/dev
/null ubuntu@
${K8S_IP} << 'EOF' 2>&1
80 PRIVATE_IP
=$
(hostname
-I |
awk '{print $1}')
82 sudo microk8s.
enable metallb
:${PRIVATE_IP}-${PRIVATE_IP}
85 # Update the certificate to allow connections from outside as well
86 ssh -T -o StrictHostKeyChecking
=no
-o UserKnownHostsFile
=/dev
/null ubuntu@
${K8S_IP} << EOF 2>&1
88 sudo sed -i "s/\#MOREIPS/IP.3 = ${K8S_IP}/g" /var/snap/microk8s/current/certs/csr.conf.template
89 cat /var/snap/microk8s/current/certs/csr.conf.template
92 # Save the credentials
93 echo ================================================================
94 echo K8s cluster credentials
:
95 echo ================================================================
97 ssh -T -o StrictHostKeyChecking
=no
-o UserKnownHostsFile
=/dev
/null ubuntu@
${K8S_IP} \
98 'sudo microk8s.config' |
sed "s/server: .*/server: https:\/\/${K8S_IP}:16443/g" \
99 |
tee ${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
102 # Create an AKS cluster with one node on Azure using a new subnet. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
103 # Required the following environment variables:
104 # - K8S_IMAGE_NAME: name of the AKS cluster to be created
105 # - RESOURCE_GROUP: name of the resource-group where the VM will be created
106 # - VNET_NAME: name of the virtual network when creating a new one or referencing an existing one
107 # - K8S_FLAVOR_NAME: the VM size to be created (e.g. "Standard_D4as_v4")
108 # IMPORTANT: required a vnet at least /16 because the it's created a new subnet with a random third byte
109 function create_azure_aks
{
110 # Gets first subnet prefix and creates a new one with random number in third byte
112 SUBNET_PREFIX
=$
(az network vnet show
--resource-group "${RESOURCE_GROUP}" --name "${VNET_NAME}" --query subnets
[0].addressPrefix
-o tsv
)
113 IFS
=.
read BYTE1 BYTE2 BYTE3 BYTE4
<<< "$SUBNET_PREFIX"
114 IFS
=/ read BYTE4 MASK
<<< "$BYTE4"
115 if [ "$MASK" -ge "24" ]; then
116 BYTE3
=$
((100 + ($RANDOM % 50) * 2))
117 PREFIX
="$BYTE1.$BYTE2.${BYTE3}.$BYTE4/$MASK"
118 SUBNET_NAME
="aks-${BYTE3}"
120 CIDR
="$BYTE1.$BYTE2.${BYTE3}.$BYTE4/$MASK"
121 DNS_IP
="$BYTE1.$BYTE2.${BYTE3}.10"
122 # Verifies that new subnet does not exist previously
123 az network vnet subnet show
--resource-group "${RESOURCE_GROUP}" --vnet-name "${VNET_NAME}" --name "$SUBNET_NAME" -o table
124 if [ "$?" -ne 0 ]; then
125 # Creates the subnet in $VNET_NAME network
126 az network vnet subnet create
--resource-group "${RESOURCE_GROUP}" --vnet-name "${VNET_NAME}" --name "$SUBNET_NAME" --address-prefixes "${PREFIX}"
127 if [ "$?" -eq 0 ]; then
128 SUBNET_ID
=$
(az network vnet subnet show
--resource-group OSM-CTIO
--vnet-name "${VNET_NAME}" --name "${SUBNET_NAME}" --query id
-o tsv
)
130 # Creates k8s cluster
131 az aks create
-y --resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --node-count 1 --node-vm-size "${K8S_FLAVOR_NAME}" --dns-service-ip "${DNS_IP}" --network-plugin kubenet --service-cidr "${CIDR}" --vnet-subnet-id "${SUBNET_ID}"
132 az aks get-credentials
--resource-group "${RESOURCE_GROUP}" --name "${K8S_IMAGE_NAME}" --admin -f ${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
138 # Create a GKE cluster with one node on GCP. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
139 # Required the following environment variables:
140 # - K8S_IMAGE_NAME: name of the GKE cluster to be created
141 # - GCP_PROJECT: name of project where the cluster will be allocated
142 # - GCP_ZONE: name of the zone (e.g. "europe-west1-b
")
143 # - GCP_MACHINE_TYPE: machine type used for the instance (e.g. "e2-standard-4
", run 'gcloud compute machine-types list')
144 # - GCP_DISK_SIZE: disk size ib GB
145 function create_gcp_gke {
146 gcloud container clusters create "${K8S_IMAGE_NAME}" --project="${GCP_PROJECT}" --zone="${GCP_ZONE}" --num-nodes=1 --machine-type="${GCP_MACHINE_TYPE}" --disk-size "${GCP_DISK_SIZE}" --enable-ip-alias --image-type "COS_CONTAINERD
"
149 # Get kubeconfig.yaml from a GKE cluster on GCP. Writes a kubeconfig.yaml file in $ROBOT_REPORT_FOLDER path.
150 function get_gke_kubeconfig {
152 FILE=${ROBOT_REPORT_FOLDER}/kubeconfig.yaml
155 echo "Creating the Kubernetes Service Account with minimal RBAC permissions.
"
156 kubectl apply -f - <<EOF
166 namespace: ${NAMESPACE}
168 apiVersion: rbac.authorization.k8s.io/v1
188 - "authorization.k8s.io
"
190 - selfsubjectaccessreviews
191 - selfsubjectrulesreviews
195 apiVersion: rbac.authorization.k8s.io/v1
196 kind: ClusterRoleBinding
200 apiGroup: rbac.authorization.k8s.io
204 - kind: ServiceAccount
206 namespace: ${NAMESPACE}
209 # Get the service account token and CA cert.
210 SA_SECRET_NAME=$(kubectl get -n ${NAMESPACE} sa/${SA} -o "jsonpath={.secrets[0]..name}")
211 # Note: service account token is stored base64-encoded in the secret but must
212 # be plaintext in kubeconfig.
213 SA_TOKEN
=$
(kubectl get
-n ${NAMESPACE} secrets/${SA_SECRET_NAME} -o "jsonpath={.data['token']}" | base64 ${BASE64_DECODE_FLAG})
214 CA_CERT=$(kubectl get -n ${NAMESPACE} secrets/${SA_SECRET_NAME} -o "jsonpath={.data['ca\.crt']}")
216 # Extract cluster IP from the current context
217 CURRENT_CONTEXT
=$
(kubectl config current-context
)
218 CURRENT_CLUSTER
=$
(kubectl config view
-o jsonpath
="{.contexts[?(@.name == \"${CURRENT_CONTEXT}\"})].context.cluster}")
219 CURRENT_CLUSTER_ADDR
=$
(kubectl config view
-o jsonpath
="{.clusters[?(@.name == \"${CURRENT_CLUSTER}\"})].cluster.server}")
226 certificate-authority-data: ${CA_CERT}
227 server: ${CURRENT_CLUSTER_ADDR}
228 name: ${CURRENT_CLUSTER}
231 cluster: ${CURRENT_CLUSTER}
232 user: ${CURRENT_CLUSTER}-${SA}
233 name: ${CURRENT_CONTEXT}
234 current-context: ${CURRENT_CONTEXT}
238 - name: ${CURRENT_CLUSTER}-${SA}
246 # Name of the new VM for K8s (adds a timestamp)
247 export K8S_IMAGE_NAME
=k8stest$
(date '+%Y%m%d%H%M')
249 # Default USE_PAAS_K8S is "FALSE"
250 if [ -z "${USE_PAAS_K8S}" ]; then
254 # Log new environment variables
255 mkdir
-p ${ROBOT_REPORT_FOLDER}
256 cat <<EOF > ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
257 export CLOUD_TYPE="${CLOUD_TYPE}"
258 export USE_PAAS_K8S="${USE_PAAS_K8S}"
261 # Branch by USE_PAAS_K8S and CLOUD_TYPE values
262 if [ "${USE_PAAS_K8S}" == "FALSE" ]; then
263 echo "Creating a new IaaS k8s cluster in ${CLOUD_TYPE}"
264 if [ "${CLOUD_TYPE}" == "azure" ]; then
267 elif [ "${CLOUD_TYPE}" == "gcp" ]; then
271 echo "CLOUD_TYPE '${CLOUD_TYPE}' not valid for IaaS"
274 # Add environment variables
275 echo "export K8S_IP=\"${K8S_IP}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
276 echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
278 # MicroK8s installation
279 install_remote_microk8s
281 echo "Creating a new PaaS k8s cluster in ${CLOUD_TYPE}"
282 if [ "${CLOUD_TYPE}" == "azure" ]; then
284 echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
285 elif [ "${CLOUD_TYPE}" == "gcp" ]; then
288 echo "export K8S_IMAGE_NAME=\"${K8S_IMAGE_NAME}\"" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
290 echo "CLOUD_TYPE '${CLOUD_TYPE}' not valid for PaaS"
294 # Add K8S_CREDENTIALS environment variable
295 echo "export K8S_CREDENTIALS=${ROBOT_REPORT_FOLDER}/kubeconfig.yaml" >> ${ROBOT_REPORT_FOLDER}/k8s_environment.rc
296 echo File with new environment was created
at ${ROBOT_REPORT_FOLDER}/k8s_environment.rc