From: garciadeblas Date: Thu, 27 Mar 2025 23:04:35 +0000 (+0100) Subject: Update stage3 to decouple VIM and cluster creation from test execution X-Git-Tag: v18.0.0~43 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F41%2F15141%2F13;p=osm%2Fdevops.git Update stage3 to decouple VIM and cluster creation from test execution Change-Id: I045b5505f072b1485bb2e630bedcb483d8096d7c Signed-off-by: garciadeblas --- diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index b9e3059c..10b8e8fa 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -63,6 +63,154 @@ properties([ //////////////////////////////////////////////////////////////////////////////////////// // Helper Functions //////////////////////////////////////////////////////////////////////////////////////// +String execute_docker_run(String tagName, String osmHostname, String environmentFile, + String portmappingfile, String kubeconfig, String clouds, + String entrypoint=null, String entrypointArgs="") { + try { + String entrypointCmd = entrypoint ? "--entrypoint ${entrypoint}" : "" + String output = sh(returnStdout: true, script: """docker run ${entrypointCmd} \ + --env OSM_HOSTNAME=${osmHostname} \ + --env-file ${environmentFile} \ + -v ${clouds}:/etc/openstack/clouds.yaml \ + -v ${kubeconfig}:/root/.kube/config \ + -v ${portmappingfile}:/root/port-mapping.yaml \ + opensourcemano/tests:${tagName} ${entrypointArgs}""") + return output + } catch (Exception e) { + println("Something happened during the execution of the shell script: ${e.message}") + throw new Exception("Docker run failed for tag '${tagName}' on host '${osmHostname}' with entrypoint '${entrypoint}' and entrypointArgs '${entrypointArgs}': ${e.message}") + } +} + +void register_etsi_vim_account( + String tagName, + String osmHostname, + String envfile=null, + String portmappingfile=null, + String kubeconfig=null, + String clouds=null +) { + String VIM_TARGET = "osm" + String VIM_MGMT_NET = "osm-ext" + String OS_PROJECT_NAME = "osm_jenkins" + String OS_AUTH_URL = "http://172.21.247.1:5000/v3" + String entrypointCmd = "/usr/bin/osm" + tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + String environmentFile = '' + if (envfile) { + environmentFile = envfile + } else { + sh(script: "touch ${tempdir}/env") + environmentFile = "${tempdir}/env" + } + int attempts = 3 + while (attempts >= 0) { + try { + println("Attempting to register VIM account (remaining attempts: ${attempts})") + withCredentials([usernamePassword(credentialsId: 'openstack-jenkins-credentials', + passwordVariable: 'OS_PASSWORD', usernameVariable: 'OS_USERNAME')]) { + String entrypointArgs = """vim-create --name ${VIM_TARGET} --user ${OS_USERNAME} \ + --password ${OS_PASSWORD} --tenant ${OS_PROJECT_NAME} \ + --auth_url ${OS_AUTH_URL} --account_type openstack --description vim \ + --config '{management_network_name: ${VIM_MGMT_NET}, dataplane_physical_net: physnet2}' || true""" + String createOutput = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("VIM Creation Output: ${createOutput}") + } + + // Check if the VIM is ENABLED + int statusChecks = 5 + while (statusChecks > 0) { + sleep(10) // Wait for 10 seconds before checking status + entrypointArgs = """vim-list --long | grep ${VIM_TARGET}""" + String vimList = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("VIM List output: ${vimList}") + if (vimList.contains("ENABLED")) { + println("VIM successfully registered and is ENABLED.") + return + } + statusChecks-- + } + + // If stuck, delete and retry + println("VIM stuck for more than 50 seconds, deleting and retrying...") + entrypointArgs = """vim-delete --force ${VIM_TARGET}""" + String deleteOutput = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("VIM Deletion Output: ${deleteOutput}") + sleep(5) + } catch (Exception e) { + println("Something happened during the execution of docker run: ${e.message}") + } + attempts-- + } + // If all attempts fail, throw an error + println("VIM failed to enter ENABLED state after multiple attempts.") + throw new Exception("VIM registration failed after multiple retries.") +} + +void register_etsi_k8s_cluster( + String tagName, + String osmHostname, + String envfile=null, + String portmappingfile=null, + String kubeconfig=null, + String clouds=null +) { + String K8S_CLUSTER_TARGET = "osm" + String VIM_TARGET = "osm" + String VIM_MGMT_NET = "osm-ext" + String K8S_CREDENTIALS = "/root/.kube/config" + String entrypointCmd = "/usr/bin/osm" + tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + String environmentFile = '' + if (envfile) { + environmentFile = envfile + } else { + sh(script: "touch ${tempdir}/env") + environmentFile = "${tempdir}/env" + } + int attempts = 3 + while (attempts >= 0) { + try { + println("Attempting to register K8s cluster (remaining attempts: ${attempts})") + String entrypointArgs = """k8scluster-add ${K8S_CLUSTER_TARGET} --creds ${K8S_CREDENTIALS} --version "v1" \ + --description "Robot-cluster" --skip-jujubundle --vim ${VIM_TARGET} \ + --k8s-nets '{net1: ${VIM_MGMT_NET}}'""" + String createOutput = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("K8s Cluster Addition Output: ${createOutput}") + + // Check if the K8s cluster is ENABLED + int statusChecks = 10 + while (statusChecks > 0) { + sleep(10) // Wait for 10 seconds before checking status + entrypointArgs = """k8scluster-list | grep ${K8S_CLUSTER_TARGET}""" + String clusterList = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("K8s Cluster List Output: ${clusterList}") + if (clusterList.contains("ENABLED")) { + println("K8s cluster successfully registered and is ENABLED.") + return + } + statusChecks-- + } + + // If stuck, delete and retry + println("K8s cluster stuck for more than 50 seconds, deleting and retrying...") + entrypointArgs = """k8scluster-show ${K8S_CLUSTER_TARGET}""" + String showOutput = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("K8s Cluster Show Output: ${showOutput}") + entrypointArgs = """k8scluster-delete ${K8S_CLUSTER_TARGET}""" + String deleteOutput = execute_docker_run(tagName, osmHostname, environmentFile, portmappingfile, kubeconfig, clouds, entrypointCmd, entrypointArgs) + println("K8s Cluster Deletion Output: ${deleteOutput}") + sleep(5) + } catch (Exception e) { + println("Something happened during the execution of docker run: ${e.message}") + } + attempts-- + } + // If all attempts fail, throw an error + println("K8s cluster failed to enter ENABLED state after multiple attempts.") + throw new Exception("K8s cluster registration failed after multiple retries.") +} + void run_robot_systest(String tagName, String testName, String osmHostname, @@ -101,7 +249,7 @@ void run_robot_systest(String tagName, } try { - withCredentials([usernamePassword(credentialsId: 'gitlab-oci-test', + withCredentials([usernamePassword(credentialsId: 'gitlab-oci-test', passwordVariable: 'OCI_REGISTRY_PSW', usernameVariable: 'OCI_REGISTRY_USR')]) { sh("""docker run --env OSM_HOSTNAME=${osmHostname} --env PROMETHEUS_HOSTNAME=${prometheusHostname} \ ${PROMETHEUS_PORT_VAR} ${JUJU_PASSWORD_VAR} --env-file ${environmentFile} \ @@ -110,7 +258,7 @@ void run_robot_systest(String tagName, -v ${clouds}:/etc/openstack/clouds.yaml -v ${osmRSAfile}:/root/osm_id_rsa \ -v ${kubeconfig}:/root/.kube/config -v ${tempdir}:/robot-systest/reports \ -v ${portmappingfile}:/root/port-mapping.yaml ${hostfilemount} opensourcemano/tests:${tagName} \ - -c -t ${testName}""") + -t ${testName}""") } } finally { try { @@ -644,6 +792,22 @@ EOF""" echo `juju gui 2>&1 | grep password | cut -d: -f2` ''' + register_etsi_vim_account( + containerName, + osmHostname, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS + ) + register_etsi_k8s_cluster( + containerName, + osmHostname, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS + ) run_robot_systest( containerName, params.ROBOT_TAG_NAME,