From: mesaj Date: Mon, 10 Nov 2025 21:05:22 +0000 (+0100) Subject: Add new e2e pipeline X-Git-Tag: v19.0.0~18 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F26%2F15526%2F9;p=osm%2Fdevops.git Add new e2e pipeline Change-Id: I5823fd54d52f32dd91d05aa79d26d9daefb0fdac Signed-off-by: mesaj --- diff --git a/jenkins/ci-pipelines/ci_e2e_tests.groovy b/jenkins/ci-pipelines/ci_e2e_tests.groovy index 58689d4c5..6984d647b 100644 --- a/jenkins/ci-pipelines/ci_e2e_tests.groovy +++ b/jenkins/ci-pipelines/ci_e2e_tests.groovy @@ -15,18 +15,1009 @@ * under the License. */ -pipeline { - agent { label 'pool' } - parameters { - string(defaultValue: env.BRANCH_NAME, description: '', name: 'GERRIT_BRANCH') +import groovy.transform.Field + +properties([ + parameters([ + // ---------------------------- + // Core: install / VM lifecycle + // ---------------------------- + string( + defaultValue: env.GERRIT_BRANCH ?: env.BRANCH_NAME ?: 'master', + description: 'Branch used to name downstream resources', + name: 'GERRIT_BRANCH' + ), + string( + defaultValue: '', + description: 'Prebuilt container tag to test (fallbacks to auto-generated name if empty)', + name: 'CONTAINER_NAME' + ), + string( + defaultValue: 'ubuntu22.04', + description: 'Glance image to use for the remote VM', + name: 'OPENSTACK_BASE_IMAGE' + ), + string( + defaultValue: 'osm.sanity', + description: 'OpenStack flavor for the remote VM', + name: 'OPENSTACK_OSM_FLAVOR' + ), + booleanParam( + defaultValue: true, + description: 'Spawn the remote VM and perform installation steps', + name: 'DO_INSTALL' + ), + booleanParam( + defaultValue: false, + description: 'Preserve VM on failure for further debugging', + name: 'SAVE_CONTAINER_ON_FAIL' + ), + booleanParam( + defaultValue: false, + description: 'Preserve VM on success', + name: 'SAVE_CONTAINER_ON_PASS' + ), + + // --------------------------------- + // Module under test / installation + // --------------------------------- + string( + defaultValue: '', + description: 'Name of the module under test', + name: 'MODULE_NAME' + ), + string( + name: 'GERRIT_REFSPEC', + defaultValue: '', + description: 'Gerrit refspec to checkout only for devops module (overrides COMMIT_ID if set)' + ), + + // ---------------------------- + // Robot / system integration + // ---------------------------- + booleanParam( + defaultValue: false, + description: 'Run Robot system integration tests after installation', + name: 'DO_ROBOT' + ), + string( + defaultValue: 'sanity', + description: 'Robot tag selection (sanity/regression/daily are common options)', + name: 'ROBOT_TAG_NAME' + ), + string( + defaultValue: '/home/jenkins/hive/robot-systest.cfg', + description: 'Robot environment file (ETSI VIM)', + name: 'ROBOT_VIM' + ), + string( + defaultValue: '/home/jenkins/hive/port-mapping-etsi-vim.yaml', + description: 'Port mapping file for SDN assist in ETSI VIM', + name: 'ROBOT_PORT_MAPPING_VIM' + ), + string( + defaultValue: '/home/jenkins/hive/etsi-vim-prometheus.json', + description: 'Prometheus configuration file in ETSI VIM', + name: 'PROMETHEUS_CONFIG_VIM' + ), + string( + defaultValue: '/home/jenkins/hive/kubeconfig.yaml', + description: 'Kubeconfig used by Robot for ETSI VIM cluster registration', + name: 'KUBECONFIG' + ), + string( + defaultValue: '/home/jenkins/hive/clouds.yaml', + description: 'OpenStack clouds.yaml used by Robot', + name: 'CLOUDS' + ), + string( + defaultValue: 'oci://osm.etsi.org:5050/devops/test', + description: 'OCI registry used by Robot system tests', + name: 'OCI_REGISTRY_URL' + ), + string( + defaultValue: '100.0', + description: '% passed Robot tests to mark the build as passed', + name: 'ROBOT_PASS_THRESHOLD' + ), + string( + defaultValue: '80.0', + description: '% passed Robot tests to mark the build as unstable (if lower, it will be failed)', + name: 'ROBOT_UNSTABLE_THRESHOLD' + ) + ]) +]) + +@Field final String HIVE_ENV_EXPORT = 'for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export $line ; done' +@Field final String INSTALLER_URL = 'https://osm-download.etsi.org/ftp/osm-19.0-nineteen/install_osm.sh' +@Field final String OPENSTACK_NET_ID = 'osm-ext' +@Field final String VCLUSTER_NAMESPACE = 'vcluster' +@Field final String VCLUSTER_NAME = 'e2e' +@Field final String ROBOT_VCLUSTER_KUBECONFIG_CONTAINER_PATH = '/robot-systest/cluster-kubeconfig.yaml' +@Field final Integer PROMETHEUS_PORT_DEFAULT = 80 +@Field final String INTERNAL_DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/' +@Field final String INTERNAL_DOCKER_REGISTRY_HOST = INTERNAL_DOCKER_REGISTRY.split('/')[0] + +// Main pipeline +node('pool') { + // Use absolute path for the SSH key to avoid tilde-expansion issues with sshCommand + final String SSH_KEY = "${env.HOME ?: '/home/jenkins'}/hive/cicd_rsa" + final String INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000' + String serverId = null + String ipAddress = '' + String kubeTmpDir = null + Map remote = null + boolean alive = false + + sh 'env' + + // Debug: list hive directory to verify SSH key presence + sh 'ls -la ~/hive || true' + + stage('Checkout') { + checkout scm + } + + def containerName = params.CONTAINER_NAME?.trim() + + // Tags for installer: + // -t : common tag for other OSM modules (stable merge build for the branch) + // -T : tag for the module under test + // -m : module name under test + def branchTag = (params.GERRIT_BRANCH ?: 'master').trim().toLowerCase().replaceAll('[^a-z0-9._-]', '-') + def commonModulesTag = "osm-${branchTag}-merge" + def testedModuleName = params.MODULE_NAME?.trim() + def testedModuleTag = containerName ?: commonModulesTag + // The `opensourcemano/tests` image is produced by the `test` module; when testing any other module, + // the tests image tag must come from the common merge build for the branch. + def testsImageTag = (testedModuleName?.equalsIgnoreCase('test') || testedModuleName?.equalsIgnoreCase('tests')) ? testedModuleTag : commonModulesTag + + Closure> buildInstallerArgs = { String registryUser, String registryPassword -> + List installArgs = ['-y'] + String installerRefspec = params.GERRIT_REFSPEC?.trim() + if (testedModuleName?.equalsIgnoreCase('devops') && installerRefspec) { + installArgs << "-S ${installerRefspec}" + } + installArgs << "-d ${registryUser}:${registryPassword}@${INTERNAL_DOCKER_REGISTRY}" + installArgs << "-p ${INTERNAL_DOCKER_PROXY}" + installArgs << "-t ${commonModulesTag}" + installArgs << "-T ${testedModuleTag}" + installArgs << "-m ${testedModuleName}" + return installArgs + } + + try { + if (params.DO_INSTALL) { +/////////////////////////////////////////////////////////////////////////////////////// +// Launch VM +/////////////////////////////////////////////////////////////////////////////////////// + stage('Spawn Remote VM') { + println('Launching new VM') + def output = runHiveCommand(""" + openstack server create --flavor ${params.OPENSTACK_OSM_FLAVOR} \ + --image ${params.OPENSTACK_BASE_IMAGE} \ + --key-name CICD \ + --property build_url=\"${BUILD_URL}\" \ + --nic net-id=${OPENSTACK_NET_ID} \ + ${containerName} + """) + + serverId = get_value('id', output) + + if (serverId == null) { + println('VM launch output:') + println(output) + throw new Exception('VM Launch failed') + } + println("Target VM is ${serverId}, waiting for IP address to be assigned") + + ipAddress = waitForServerIp(serverId) + println("Waiting for VM at ${ipAddress} to be reachable") + + remote = [ + name: containerName ?: "osm-e2e-${BUILD_NUMBER}", + host: ipAddress, + user: 'ubuntu', + identityFile: SSH_KEY, + allowAnyHosts: true, + logLevel: 'INFO', + pty: true + ] + + alive = false + timeout(time: 1, unit: 'MINUTES') { + while (!alive) { + def sshStatus = sh( + returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "-o ConnectTimeout=5 ubuntu@${ipAddress} 'echo Alive'") + alive = (sshStatus == 0) + } + } + println('VM is ready and accepting ssh connections') + ////////////////////////////////////////////////////////////////////////////////////////////// + println('Applying sshd config workaround for Ubuntu 22.04 and old jsch client in Jenkins (via native ssh)...') + sh """ssh -T -i ${SSH_KEY} \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ubuntu@${ipAddress} \"echo HostKeyAlgorithms +ssh-rsa | sudo tee -a /etc/ssh/sshd_config\" + """ + sh """ssh -T -i ${SSH_KEY} \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ubuntu@${ipAddress} \"echo PubkeyAcceptedKeyTypes +ssh-rsa | sudo tee -a /etc/ssh/sshd_config\" + """ + sh """ssh -T -i ${SSH_KEY} \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ubuntu@${ipAddress} \"sudo systemctl restart sshd\" + """ + ////////////////////////////////////////////////////////////////////////////////////////////// + } // stage("Spawn Remote VM") +/////////////////////////////////////////////////////////////////////////////////////// +// Checks before installation +/////////////////////////////////////////////////////////////////////////////////////// + stage('Checks before installation') { + if (!ipAddress?.trim()) { + error('Missing VM IP address, cannot run pre-installation checks') + } + + sshCommand remote: remote, command: 'cloud-init status --wait' + sshCommand remote: remote, command: 'sudo apt-get -y update' + sshCommand remote: remote, command: 'sudo apt-get -y install chrony' + sshCommand remote: remote, command: 'sudo service chrony stop' + sshCommand remote: remote, command: 'sudo chronyd -vq' + sshCommand remote: remote, command: 'sudo service chrony start' + } // stage("Checks before installation") +/////////////////////////////////////////////////////////////////////////////////////// +// Install +/////////////////////////////////////////////////////////////////////////////////////// + stage('Install') { + if (!ipAddress?.trim()) { + error('Missing VM IP address, cannot run installation steps') + } + + sshCommand remote: remote, command: """ + wget ${INSTALLER_URL} + chmod +x ./install_osm.sh + sed -i '1 i\\export PATH=/snap/bin:\$PATH' ~/.bashrc + """ + + + Map gitlabCredentialsMap = [$class: 'UsernamePasswordMultiBinding', + credentialsId: 'gitlab-registry', + usernameVariable: 'USERNAME', + passwordVariable: 'PASSWORD'] + withCredentials([gitlabCredentialsMap]) { + List installArgs = buildInstallerArgs(USERNAME, PASSWORD) + + String installCmd = "./install_osm.sh ${installArgs.join(' ')}" + sshCommand remote: remote, command: """ + ${installCmd} + """ + } + } // stage("Install") + +/////////////////////////////////////////////////////////////////////////////////////// +// Health check of installed OSM in remote VM +/////////////////////////////////////////////////////////////////////////////////////// + stage('OSM Health') { + if (!ipAddress?.trim()) { + error('Missing VM IP address, cannot run OSM health checks') + } + if (!remote) { + error('Missing remote target, cannot run OSM health checks') + } + + timeout(time: 5, unit: 'MINUTES') { + String osmHostname = "nbi.${ipAddress}.nip.io" + sshCommand remote: remote, command: """ + export PATH=\"\\$PATH:\\$HOME/.local/bin\" + OSM_HOSTNAME=\"${osmHostname}\" osm vim-list + """ + } + } // stage('OSM Health') + +/////////////////////////////////////////////////////////////////////////////////////// +// Get OSM Kubeconfig and store it for future usage (Robot/vCluster) +/////////////////////////////////////////////////////////////////////////////////////// + if (params.DO_ROBOT) { + stage('OSM Get kubeconfig') { + kubeTmpDir = pwd(tmp: true) + env.OSM_KUBECONFIG_PATH = "${kubeTmpDir}/osm_config" + env.VCLUSTER_KUBECONFIG_PATH = "${kubeTmpDir}/vcluster_config" + + sshGet remote: remote, + from: "/home/ubuntu/.kube/config", + into: env.OSM_KUBECONFIG_PATH, + override: true + sh "chmod 600 ${env.OSM_KUBECONFIG_PATH}" + sh "test -s ${env.OSM_KUBECONFIG_PATH}" + // Debug: show the Kubernetes API endpoint used by the kubeconfig. + // (k3s defaults to 127.0.0.1:6443, which is not reachable from the Jenkins agent container) + sh "grep -nE '^\\s*server:' ${env.OSM_KUBECONFIG_PATH} || true" + } // stage('OSM Get kubeconfig') + +/////////////////////////////////////////////////////////////////////////////////////// +// Create vCluster for GitOps/Robot test execution +/////////////////////////////////////////////////////////////////////////////////////// + stage('Create vCluster') { + println("Creating vcluster ${VCLUSTER_NAME} in namespace ${VCLUSTER_NAMESPACE}") + dockerLoginInternalRegistry() + create_vcluster(INTERNAL_DOCKER_REGISTRY, testsImageTag, env.OSM_KUBECONFIG_PATH, env.VCLUSTER_KUBECONFIG_PATH, VCLUSTER_NAME, VCLUSTER_NAMESPACE) + sh "chmod 600 ${env.VCLUSTER_KUBECONFIG_PATH}" + sh "test -s ${env.VCLUSTER_KUBECONFIG_PATH}" + } // stage('Create vCluster') + +/////////////////////////////////////////////////////////////////////////////////////// +// Execute Robot tests +/////////////////////////////////////////////////////////////////////////////////////// + stage('System Integration Test') { + String prometheusHostname = "prometheus.${ipAddress}.nip.io" + Integer prometheusPort = PROMETHEUS_PORT_DEFAULT + String osmHostnameRobot = "nbi.${ipAddress}.nip.io:443" + + register_etsi_vim_account( + INTERNAL_DOCKER_REGISTRY, + testsImageTag, + osmHostnameRobot, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS, + params.PROMETHEUS_CONFIG_VIM + ) + register_etsi_k8s_cluster( + INTERNAL_DOCKER_REGISTRY, + testsImageTag, + osmHostnameRobot, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS, + params.PROMETHEUS_CONFIG_VIM + ) + + // IMPORTANT: tests expect the vcluster kubeconfig at this container path. + String robotVclusterKubeconfigPath = ROBOT_VCLUSTER_KUBECONFIG_CONTAINER_PATH + run_robot_systest( + INTERNAL_DOCKER_REGISTRY, + testsImageTag, + params.ROBOT_TAG_NAME, + osmHostnameRobot, + prometheusHostname, + prometheusPort, + params.OCI_REGISTRY_URL, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS, + null, + SSH_KEY, + params.ROBOT_PASS_THRESHOLD, + params.ROBOT_UNSTABLE_THRESHOLD, + // extraEnvVars map of extra environment variables + ['CLUSTER_KUBECONFIG_CREDENTIALS': robotVclusterKubeconfigPath], + // extraVolMounts map of extra volume mounts + [(env.VCLUSTER_KUBECONFIG_PATH): robotVclusterKubeconfigPath] + ) + } // stage('System Integration Test') + } else { + println('Skipping kubeconfig/vcluster steps because DO_ROBOT is set to false') + } + } else { + println('Skipping VM spawn because DO_INSTALL is set to false') + } + } finally { + stage('Archive Logs') { + if (params.DO_INSTALL && remote) { + try { + archiveLogs(remote) + } catch (Exception e) { + println("Archive logs failed: ${e.message}") + } + } else { + println('No remote target to collect logs from') + } + } + + stage('Cleanup') { + // Always attempt to cleanup temp kubeconfig directory if created. + if (kubeTmpDir?.trim()) { + sh "rm -rf ${kubeTmpDir} || true" + kubeTmpDir = null + } + + if (!params.DO_INSTALL || serverId == null) { + println('No VM to cleanup') + return + } + + String buildState = currentBuild.currentResult ?: 'SUCCESS' + boolean buildFailed = buildState == 'FAILURE' + + boolean deleteVm = true + if (buildFailed && params.SAVE_CONTAINER_ON_FAIL) { + deleteVm = false + } + if (!buildFailed && params.SAVE_CONTAINER_ON_PASS) { + deleteVm = false + } + + if (deleteVm) { + println("Deleting VM: ${serverId}") + try { + runHiveCommand(""" + openstack server delete ${serverId} + """) + } catch (Exception e) { + // Avoid masking an earlier failure with cleanup failure. + println("VM delete failed: ${e.message}") + } + } else { + println("Preserving VM ${serverId} (build state: ${buildState})") + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////// +// Helper Classes & Functions (ported from ci_stage_3.groovy) +// Placed below the pipeline for readability. +//////////////////////////////////////////////////////////////////////////////////////// + +/** Usage: + * def dr = new DockerRunner(this) + * stdout = dr.run( + * image : "${INTERNAL_DOCKER_REGISTRY}opensourcemano/tests:${tag}", + * entry : "/usr/bin/osm", // optional + * envVars : [ "OSM_HOSTNAME=${host}" ], + * envFile : myEnv, + * mounts : [ + * "${clouds}:/etc/openstack/clouds.yaml", + * "${kubeconfig}:/root/.kube/config" + * ], + * cmd : "vim-create --name osm …" + * ) + */ +class DockerRunner implements Serializable { + def steps + DockerRunner(def steps) { this.steps = steps } + + /** Returns stdout (trimmed) if returnStdout is true; throws Exception on non-zero exit */ + String run(Map args = [:]) { + def returnStdout = args.remove('returnStdout') ?: false + def envFile = args.envFile ?: '' + def entry = args.entry ? "--entrypoint ${args.entry}" : '' + def mounts = (args.mounts ?: []) + .findAll { it && it.trim() } + .collect { "-v ${it}" } + .join(' ') + def envs = (args.envVars ?: []) + .findAll { it && it.trim() } + .collect { "--env ${it}" } + .join(' ') + def image = args.image ?: '' + def cmd = args.cmd ?: '' + def fullCmd = "docker run --rm ${entry} ${envs} ${envFile ? "--env-file ${envFile}" : ''} ${mounts} ${image} ${cmd}".trim() + + def result = null + try { + if (returnStdout) { + result = steps.sh(returnStdout: true, script: fullCmd).trim() + } else { + steps.sh(script: fullCmd) + } + } catch (Exception ex) { + throw new Exception("docker run failed -> ${ex.message}") + } finally { + steps.echo("Command executed: ${fullCmd}") + } + return result ?: '' + } +} + +/* ------------------------------------------------------------------- + * create_vcluster – spin up a vcluster in the target OSM cluster + * @params: + * tagName - The OSM test docker image tag to use + * kubeconfigPath - The path of the OSM kubernetes master configuration file + * vclusterKubeconfigOutPath - Output path for the vcluster kubeconfig + * vclusterName - Name of the vcluster + * vclusterNamespace - Namespace for the vcluster + * ------------------------------------------------------------------- */ +void create_vcluster(String dockerRegistryUrl, String tagName, String kubeconfigPath, String vclusterKubeconfigOutPath, String vclusterName, String vclusterNamespace) { + def dr = new DockerRunner(this) + def mounts = ["${kubeconfigPath}:/root/.kube/config"] + def envs = ["KUBECONFIG=/root/.kube/config"] + def image = "${dockerRegistryUrl}opensourcemano/tests:${tagName}" + + // 1) create vcluster namespace + println("vcluster: ensuring namespace '${vclusterNamespace}' exists") + dr.run( + image: image, + entry: 'kubectl', + envVars: envs, + mounts: mounts, + cmd: "create namespace ${vclusterNamespace} || true" + ) + + // 2) create vcluster (no connect) + println("vcluster: creating '${vclusterName}' (no connect)") + dr.run( + image: image, + entry: 'vcluster', + envVars: envs, + mounts: mounts, + cmd: "create ${vclusterName} -n ${vclusterNamespace} --connect=false -f /etc/vcluster.yaml" + ) + + // 3) poll until Status is Running + int maxWaitMinutes = 10 + long deadline = System.currentTimeMillis() + (maxWaitMinutes * 60 * 1000) + boolean running = false + String lastOut = '' + + println("vcluster: waiting for '${vclusterName}' to reach status 'Running' (timeout: ${maxWaitMinutes} minutes)") + while (System.currentTimeMillis() < deadline) { + try { + lastOut = dr.run( + returnStdout: true, + image: image, + entry: '/bin/sh', + envVars: envs, + mounts: mounts, + cmd: '''-c "vcluster list --output json | jq -r ".[] | select(.Name==\"''' + vclusterName + '''\") | .Status\"''' + ).trim() + } catch (Exception e) { + println("Polling command failed: ${e.message}. Will retry.") + lastOut = "Error: ${e.message}" + } + + println("Polling for vcluster status. Current status: '${lastOut}'") + if (lastOut == 'Running') { + running = true + break + } + sleep 10 + } + + if (!running) { + println("vcluster status after timeout: ${lastOut}") + throw new Exception("vcluster '${vclusterName}' did not reach 'Running' state within ${maxWaitMinutes} minutes.") + } + + // 4) get vcluster kubeconfig + String outPath = vclusterKubeconfigOutPath ?: "${WORKSPACE}/kubeconfig/vcluster_config" + // Ensure destination directory exists on the Jenkins agent before relying on shell redirection. + String outDir = outPath.contains('/') ? outPath.substring(0, outPath.lastIndexOf('/')) : '.' + sh "mkdir -p ${outDir}" + println("vcluster: exporting kubeconfig to '${outPath}'") + dr.run( + image: image, + entry: 'vcluster', + envVars: envs, + mounts: mounts, + cmd: "connect ${vclusterName} -n ${vclusterNamespace} --server ${vclusterName}.${vclusterNamespace}.svc.cluster.local:443 --print > ${outPath}" + ) +} + +void retryWithDocker(int maxAttempts, int delaySeconds, Closure action) { + int attempts = maxAttempts + while (attempts >= 0) { + try { + if (action()) return + } catch (Exception e) { + println("Attempt failed: ${e.message}") + } + println("Retrying... (${attempts} attempts left)") + sleep delaySeconds + attempts-- } - stages { - stage('TEST') { - agent { label 'pool' } - steps { - echo "HELLO" + throw new Exception("Operation failed after ${maxAttempts} retries") +} + +void register_etsi_vim_account( + String dockerRegistryUrl, + String tagName, + String osmHostname, + String envfile = null, + String portmappingfile = null, + String kubeconfig = null, + String clouds = null, + String prometheusconfigfile = null +) { + String VIM_TARGET = "osm" + String VIM_MGMT_NET = "osm-ext" + String OS_PROJECT_NAME = "osm_jenkins" + String OS_AUTH_URL = "http://172.21.247.1:5000/v3" + String entrypointCmd = "/usr/bin/osm" + + def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + String environmentFile = envfile ?: "${tempdir}/env" + if (!envfile) { + sh(script: "touch ${environmentFile}") + } + + retryWithDocker(3, 10) { + def dr = new DockerRunner(this) + try { + println("Attempting to register VIM account") + withCredentials([usernamePassword(credentialsId: 'openstack-jenkins-credentials', + passwordVariable: 'OS_PASSWORD', usernameVariable: 'OS_USERNAME')]) { + String entrypointArgs = """vim-create --name ${VIM_TARGET} --user ${OS_USERNAME} \ + --password ${OS_PASSWORD} --tenant ${OS_PROJECT_NAME} \ + --auth_url ${OS_AUTH_URL} --account_type openstack --description vim \ + --prometheus_config_file /root/etsi-vim-prometheus.json \ + --config '{management_network_name: ${VIM_MGMT_NET}, dataplane_physical_net: physnet2}' || true""" + dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: entrypointArgs, + returnStdout: true + ) + } + + // Check if the VIM is ENABLED + int statusChecks = 5 + while (statusChecks > 0) { + sleep 10 + String vimList = dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: "vim-list --long | grep ${VIM_TARGET}", + returnStdout: true + ) + if (vimList.contains("ENABLED")) { + println("VIM successfully registered and is ENABLED.") + return true + } + statusChecks-- } + } catch (Exception e) { + println("VIM registration check failed: ${e.message}") } + + // If we get here, VIM is not enabled or creation failed. cleanup and retry. + println("VIM not enabled, deleting and retrying...") + dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: "vim-delete --force ${VIM_TARGET}", + returnStdout: true + ) + return false } } +void register_etsi_k8s_cluster( + String dockerRegistryUrl, + String tagName, + String osmHostname, + String envfile = null, + String portmappingfile = null, + String kubeconfig = null, + String clouds = null, + String prometheusconfigfile = null +) { + String K8S_CLUSTER_TARGET = "osm" + String VIM_TARGET = "osm" + String VIM_MGMT_NET = "osm-ext" + String K8S_CREDENTIALS = "/root/.kube/config" + String entrypointCmd = "/usr/bin/osm" + + def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + String environmentFile = envfile ?: "${tempdir}/env" + if (!envfile) { + sh(script: "touch ${environmentFile}") + } + + retryWithDocker(3, 10) { + def dr = new DockerRunner(this) + try { + println("Attempting to register K8s cluster") + dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: """k8scluster-add ${K8S_CLUSTER_TARGET} --creds ${K8S_CREDENTIALS} --version \"v1\" \ + --description \"Robot-cluster\" --skip-jujubundle --vim ${VIM_TARGET} \ + --k8s-nets '{net1: ${VIM_MGMT_NET}}'""", + returnStdout: true + ) + + // Check if the K8s cluster is ENABLED + int statusChecks = 10 + while (statusChecks > 0) { + sleep 10 + String clusterList = dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: "k8scluster-list | grep ${K8S_CLUSTER_TARGET}", + returnStdout: true + ) + if (clusterList.contains("ENABLED")) { + println("K8s cluster successfully registered and is ENABLED.") + return true + } + statusChecks-- + } + } catch (Exception e) { + println("K8s cluster registration check failed: ${e.message}") + } + + // If we get here, cluster is not enabled or creation failed. cleanup and retry. + println("K8s cluster not enabled, deleting and retrying...") + dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + entry: entrypointCmd, + envVars: ["OSM_HOSTNAME=${osmHostname}"], + envFile: environmentFile, + mounts: [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null + ].findAll { it != null }, + cmd: "k8scluster-delete ${K8S_CLUSTER_TARGET}" + ) + return false + } +} + +void run_robot_systest( + String dockerRegistryUrl, + String tagName, + String testName, + String osmHostname, + String prometheusHostname, + Integer prometheusPort = null, + String ociRegistryUrl = null, + String envfile = null, + String portmappingfile = null, + String kubeconfig = null, + String clouds = null, + String hostfile = null, + String osmRSAfile = null, + String passThreshold = '0.0', + String unstableThreshold = '0.0', + Map extraEnvVars = null, + Map extraVolMounts = null +) { + def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + String environmentFile = envfile ?: "${tempdir}/env" + if (!envfile) { + sh(script: "touch ${environmentFile}") + } + + def prometheusPortVar = prometheusPort != null ? "PROMETHEUS_PORT=${prometheusPort}" : null + def hostfilemount = hostfile ? "${hostfile}:/etc/hosts" : null + + try { + withCredentials([usernamePassword(credentialsId: 'gitlab-oci-test', + passwordVariable: 'OCI_REGISTRY_PSW', usernameVariable: 'OCI_REGISTRY_USR')]) { + def baseEnvVars = [ + "OSM_HOSTNAME=${osmHostname}", + "PROMETHEUS_HOSTNAME=${prometheusHostname}", + prometheusPortVar, + ociRegistryUrl ? "OCI_REGISTRY_URL=${ociRegistryUrl}" : null, + "OCI_REGISTRY_USER=${OCI_REGISTRY_USR}", + "OCI_REGISTRY_PASSWORD=${OCI_REGISTRY_PSW}" + ].findAll { it != null } + + def baseMounts = [ + clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null, + osmRSAfile ? "${osmRSAfile}:/root/osm_id_rsa" : null, + kubeconfig ? "${kubeconfig}:/root/.kube/config" : null, + "${tempdir}:/robot-systest/reports", + portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null, + hostfilemount + ].findAll { it != null } + + def extraEnvVarsList = extraEnvVars?.collect { key, value -> "${key}=${value}" } ?: [] + def extraVolMountsList = extraVolMounts?.collect { hostPath, containerPath -> "${hostPath}:${containerPath}" } ?: [] + + def dr = new DockerRunner(this) + dr.run( + image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}", + envVars: baseEnvVars + extraEnvVarsList, + envFile: "${environmentFile}", + mounts: baseMounts + extraVolMountsList, + cmd: "-t ${testName}" + ) + } + } finally { + // Best-effort publish Robot results from tempdir into workspace + sh("cp ${tempdir}/*.xml . 2>/dev/null || true") + sh("cp ${tempdir}/*.html . 2>/dev/null || true") + + def outputDirectory = sh(returnStdout: true, script: 'pwd').trim() + sh("command -v tree >/dev/null 2>&1 && tree ${outputDirectory} || ls -la ${outputDirectory}") + + try { + step([ + $class: 'RobotPublisher', + outputPath: "${outputDirectory}", + outputFileName: '*.xml', + disableArchiveOutput: false, + reportFileName: 'report.html', + logFileName: 'log.html', + passThreshold: passThreshold, + unstableThreshold: unstableThreshold, + otherFiles: '*.png' + ]) + } catch (Exception e) { + println("RobotPublisher failed: ${e.message}") + } + } +} + +String get_value(String key, String output) { + for (String line : output.split('\n')) { + def data = line.split('\\|') + if (data.length > 1 && data[1].trim() == key) { + return data[2].trim() + } + } + return null +} + +void dockerLoginInternalRegistry() { + withCredentials([usernamePassword(credentialsId: 'gitlab-registry', + passwordVariable: 'REGISTRY_PASSWORD', usernameVariable: 'REGISTRY_USERNAME')]) { + sh """ + set -e + echo "${REGISTRY_PASSWORD}" | docker login ${INTERNAL_DOCKER_REGISTRY_HOST} -u "${REGISTRY_USERNAME}" --password-stdin + """ + } +} + +String withHiveEnv(String commandBody) { + """#!/bin/sh -e +${HIVE_ENV_EXPORT} +${commandBody.stripIndent()} +""" +} + +String runHiveCommand(String commandBody) { + sh(returnStdout: true, script: withHiveEnv(commandBody)).trim() +} + +String waitForServerIp(String id) { + String addr = '' + timeout(time: 5, unit: 'MINUTES') { + waitUntil { + def showOutput = runHiveCommand(""" + openstack server show ${id} + """) + def rawAddress = get_value('addresses', showOutput) + if (rawAddress) { + addr = rawAddress.split('=')[1] + return true + } + sleep 5 + return false + } + } + return addr +} + +// Collect logs from the remote VM and archive them in Jenkins +void archiveLogs(Map remoteTarget) { + sshCommand remote: remoteTarget, command: '''mkdir -p logs/dags logs/vcluster logs/flux-system logs/events logs/system''' + + // Collect Kubernetes events + sshCommand remote: remoteTarget, command: ''' + echo "Extracting Kubernetes events" + kubectl get events --all-namespaces --sort-by='.lastTimestamp' -o wide > logs/events/k8s-events.log 2>&1 || true + kubectl get events -n osm --sort-by='.lastTimestamp' -o wide > logs/events/osm-events.log 2>&1 || true + kubectl get events -n vcluster --sort-by='.lastTimestamp' -o wide > logs/events/vcluster-events.log 2>&1 || true + kubectl get events -n flux-system --sort-by='.lastTimestamp' -o wide > logs/events/flux-system-events.log 2>&1 || true + ''' + + // Collect host logs and system info + sshCommand remote: remoteTarget, command: ''' + echo "Collect system logs" + if command -v journalctl >/dev/null; then + journalctl > logs/system/system.log + fi + + for entry in syslog messages; do + [ -e "/var/log/${entry}" ] && cp -f /var/log/${entry} logs/system/"${entry}.log" + done + + echo "Collect active services" + case "$(cat /proc/1/comm)" in + systemd) + systemctl list-units > logs/system/services.txt 2>&1 + ;; + *) + service --status-all >> logs/system/services.txt 2>&1 + ;; + esac + + top -b -n 1 > logs/system/top.txt 2>&1 + ps fauxwww > logs/system/ps.txt 2>&1 + ''' + + // Collect OSM namespace workloads + sshCommand remote: remoteTarget, command: ''' + for deployment in `kubectl -n osm get deployments | grep -v operator | grep -v NAME| awk '{print $1}'`; do + echo "Extracting log for $deployment" + kubectl -n osm logs deployments/$deployment --timestamps=true --all-containers 2>&1 > logs/$deployment.log || true + done + ''' + sshCommand remote: remoteTarget, command: ''' + for statefulset in `kubectl -n osm get statefulsets | grep -v operator | grep -v NAME| awk '{print $1}'`; do + echo "Extracting log for $statefulset" + kubectl -n osm logs statefulsets/$statefulset --timestamps=true --all-containers 2>&1 > logs/$statefulset.log || true + done + ''' + sshCommand remote: remoteTarget, command: ''' + schedulerPod="$(kubectl get pods -n osm | grep osm-scheduler| awk '{print $1; exit}')"; \ + echo "Extracting logs from Airflow DAGs from pod ${schedulerPod}"; \ + kubectl -n osm cp ${schedulerPod}:/opt/airflow/logs/scheduler/latest/dags logs/dags -c scheduler 2>&1 || true + ''' + + // Collect vcluster and flux-system namespace logs + sshCommand remote: remoteTarget, command: ''' + echo "Extracting logs from vcluster namespace" + for pod in `kubectl get pods -n vcluster | grep -v NAME | awk '{print $1}'`; do + echo "Extracting log for vcluster pod: $pod" + kubectl logs -n vcluster $pod --timestamps=true --all-containers 2>&1 > logs/vcluster/$pod.log || true + done + ''' + sshCommand remote: remoteTarget, command: ''' + echo "Extracting logs from flux-system namespace" + for pod in `kubectl get pods -n flux-system | grep -v NAME | awk '{print $1}'`; do + echo "Extracting log for flux-system pod: $pod" + kubectl logs -n flux-system $pod --timestamps=true --all-containers 2>&1 > logs/flux-system/$pod.log || true + done + ''' + + sh 'rm -rf logs' + sshCommand remote: remoteTarget, command: '''ls -al logs logs/vcluster logs/events logs/flux-system logs/system''' + sshGet remote: remoteTarget, from: 'logs', into: '.', override: true + archiveArtifacts artifacts: 'logs/*.log, logs/dags/*.log, logs/vcluster/*.log, logs/events/*.log, logs/flux-system/*.log, logs/system/**' +} diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 8a4cf6080..f03dbe22b 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -1019,7 +1019,6 @@ node("${params.NODE}") { /////////////////////////////////////////////////////////////////////////////////////// // Create vCluster for GitOps test execution -/////////////////////////////////////////////////////////////////////////////////////// stage('Create vCluster') { // create an isolated vcluster for the E2E tests create_vcluster(containerName, env.OSM_KUBECONFIG_PATH)