jenkins/ci-pipelines/ci_e2e_tests.groovy

/* Copyright ETSI Contributors and Others
 *
 * All Rights Reserved.
 *
 *   Licensed under the Apache License, Version 2.0 (the "License"); you may
 *   not use this file except in compliance with the License. You may obtain
 *   a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 *   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 *   License for the specific language governing permissions and limitations
 *   under the License.
 */

import groovy.transform.Field

properties([
    parameters([
        // ----------------------------
        // Core: install / VM lifecycle
        // ----------------------------
        string(
            defaultValue: env.GERRIT_BRANCH ?: env.BRANCH_NAME ?: 'master',
            description: 'Branch used to name downstream resources',
            name: 'GERRIT_BRANCH'
        ),
        string(
            defaultValue: '',
            description: 'Prebuilt container tag to test (fallbacks to auto-generated name if empty)',
            name: 'CONTAINER_NAME'
        ),
        string(
            defaultValue: 'ubuntu24.04',
            description: 'Glance image to use for the remote VM',
            name: 'OPENSTACK_BASE_IMAGE'
        ),
        string(
            defaultValue: 'osm.sanity',
            description: 'OpenStack flavor for the remote VM',
            name: 'OPENSTACK_OSM_FLAVOR'
        ),
        booleanParam(
            defaultValue: true,
            description: 'Spawn the remote VM and perform installation steps',
            name: 'DO_INSTALL'
        ),
        booleanParam(
            defaultValue: false,
            description: 'Preserve VM on failure for further debugging',
            name: 'SAVE_CONTAINER_ON_FAIL'
        ),
        booleanParam(
            defaultValue: false,
            description: 'Preserve VM on success',
            name: 'SAVE_CONTAINER_ON_PASS'
        ),

        // ---------------------------------
        // Module under test / installation
        // ---------------------------------
        string(
            defaultValue: '',
            description: 'Name of the module under test',
            name: 'MODULE_NAME'
        ),
        string(
            name: 'GERRIT_REFSPEC',
            defaultValue: '',
            description: 'Gerrit refspec to checkout only for devops module (overrides COMMIT_ID if set)'
        ),

        // ----------------------------
        // Robot / system integration
        // ----------------------------
        booleanParam(
            defaultValue: false,
            description: 'Run Robot system integration tests after installation',
            name: 'DO_ROBOT'
        ),
        string(
            defaultValue: 'sanity',
            description: 'Robot tag selection (sanity/regression/daily are common options)',
            name: 'ROBOT_TAG_NAME'
        ),
        string(
            defaultValue: '/home/jenkins/hive/robot-systest.cfg',
            description: 'Robot environment file (ETSI VIM)',
            name: 'ROBOT_VIM'
        ),
        string(
            defaultValue: '/home/jenkins/hive/port-mapping-etsi-vim.yaml',
            description: 'Port mapping file for SDN assist in ETSI VIM',
            name: 'ROBOT_PORT_MAPPING_VIM'
        ),
        string(
            defaultValue: '/home/jenkins/hive/etsi-vim-prometheus.json',
            description: 'Prometheus configuration file in ETSI VIM',
            name: 'PROMETHEUS_CONFIG_VIM'
        ),
        string(
            defaultValue: '/home/jenkins/hive/kubeconfig.yaml',
            description: 'Kubeconfig used by Robot for ETSI VIM cluster registration',
            name: 'KUBECONFIG'
        ),
        string(
            defaultValue: '/home/jenkins/hive/clouds.yaml',
            description: 'OpenStack clouds.yaml used by Robot',
            name: 'CLOUDS'
        ),
        string(
            defaultValue: 'oci://osm.etsi.org:5050/devops/test',
            description: 'OCI registry used by Robot system tests',
            name: 'OCI_REGISTRY_URL'
        ),
        string(
            defaultValue: '100.0',
            description: '% passed Robot tests to mark the build as passed',
            name: 'ROBOT_PASS_THRESHOLD'
        ),
        string(
            defaultValue: '80.0',
            description: '% passed Robot tests to mark the build as unstable (if lower, it will be failed)',
            name: 'ROBOT_UNSTABLE_THRESHOLD'
        )
    ])
])

@Field final String HIVE_ENV_EXPORT = 'for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export $line ; done'
@Field final String INSTALLER_URL = 'https://osm-download.etsi.org/ftp/osm-19.0-nineteen/install_osm.sh'
@Field final String OPENSTACK_NET_ID = 'osm-ext'
@Field final String VCLUSTER_NAMESPACE = 'vcluster'
@Field final String VCLUSTER_NAME = 'e2e'
@Field final String ROBOT_VCLUSTER_KUBECONFIG_CONTAINER_PATH = '/robot-systest/cluster-kubeconfig.yaml'
@Field final Integer PROMETHEUS_PORT_DEFAULT = 80
@Field final String INTERNAL_DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/'
@Field final String INTERNAL_DOCKER_REGISTRY_HOST = INTERNAL_DOCKER_REGISTRY.split('/')[0]

// Main pipeline
node('pool') {
    // Use absolute path for the SSH key to avoid tilde-expansion issues with sshCommand
    final String SSH_KEY = "${env.HOME ?: '/home/jenkins'}/hive/cicd_rsa"
    final String INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000'
    String serverId = null
    String ipAddress = ''
    String kubeTmpDir = null
    Map remote = null
    boolean alive = false

    sh 'env'

    // Debug: list hive directory to verify SSH key presence
    sh 'ls -la ~/hive || true'

    stage('Checkout') {
        checkout scm
    }

    def containerName = params.CONTAINER_NAME?.trim()

    // Tags for installer:
    // -t : common tag for other OSM modules (stable merge build for the branch)
    // -T : tag for the module under test
    // -m : module name under test
    def branchTag = (params.GERRIT_BRANCH ?: 'master').trim().toLowerCase().replaceAll('[^a-z0-9._-]', '-')
    def commonModulesTag = "osm-${branchTag}-merge"
    def testedModuleName = params.MODULE_NAME?.trim()
    def testedModuleTag = containerName ?: commonModulesTag
    // The `opensourcemano/tests` image is produced by the `test` module; when testing any other module,
    // the tests image tag must come from the common merge build for the branch.
    def testsImageTag = (testedModuleName?.equalsIgnoreCase('test') || testedModuleName?.equalsIgnoreCase('tests')) ? testedModuleTag : commonModulesTag

    Closure<List<String>> buildInstallerArgs = { String registryUser, String registryPassword ->
        List<String> installArgs = ['-y']
        String installerRefspec = params.GERRIT_REFSPEC?.trim()
        if (testedModuleName?.equalsIgnoreCase('devops') && installerRefspec) {
            installArgs << "-S ${installerRefspec}"
        }
        installArgs << "-d ${registryUser}:${registryPassword}@${INTERNAL_DOCKER_REGISTRY}"
        installArgs << "-p ${INTERNAL_DOCKER_PROXY}"
        installArgs << "-t ${commonModulesTag}"
        installArgs << "-T ${testedModuleTag}"
        installArgs << "-m ${testedModuleName}"
        return installArgs
    }

    try {
        if (params.DO_INSTALL) {
///////////////////////////////////////////////////////////////////////////////////////
// Launch VM
///////////////////////////////////////////////////////////////////////////////////////
            stage('Spawn Remote VM') {
                println('Launching new VM')
                def output = runHiveCommand("""
                    openstack server create --flavor ${params.OPENSTACK_OSM_FLAVOR} \
                                            --image ${params.OPENSTACK_BASE_IMAGE} \
                                            --key-name CICD \
                                            --property build_url=\"${BUILD_URL}\" \
                                            --nic net-id=${OPENSTACK_NET_ID} \
                                            ${containerName}
                """)

                serverId = get_value('id', output)

                if (serverId == null) {
                    println('VM launch output:')
                    println(output)
                    throw new Exception('VM Launch failed')
                }
                println("Target VM is ${serverId}, waiting for IP address to be assigned")

                ipAddress = waitForServerIp(serverId)
                println("Waiting for VM at ${ipAddress} to be reachable")

                remote = [
                    name: containerName ?: "osm-e2e-${BUILD_NUMBER}",
                    host: ipAddress,
                    user: 'ubuntu',
                    identityFile: SSH_KEY,
                    allowAnyHosts: true,
                    logLevel: 'INFO',
                    pty: true
                ]

                alive = false
                timeout(time: 1, unit: 'MINUTES') {
                    while (!alive) {
                        def sshStatus = sh(
                            returnStatus: true,
                            script: "ssh -T -i ${SSH_KEY} " +
                                "-o StrictHostKeyChecking=no " +
                                "-o UserKnownHostsFile=/dev/null " +
                                "-o ConnectTimeout=5 ubuntu@${ipAddress} 'echo Alive'")
                        alive = (sshStatus == 0)
                    }
                }
                println('VM is ready and accepting ssh connections')
                //////////////////////////////////////////////////////////////////////////////////////////////
                println('Applying sshd config workaround for Ubuntu 22.04 and old jsch client in Jenkins (via native ssh)...')
                sh """ssh -T -i ${SSH_KEY} \
                    -o StrictHostKeyChecking=no \
                    -o UserKnownHostsFile=/dev/null \
                    ubuntu@${ipAddress} \"echo HostKeyAlgorithms +ssh-rsa | sudo tee -a /etc/ssh/sshd_config\"
                """
                sh """ssh -T -i ${SSH_KEY} \
                    -o StrictHostKeyChecking=no \
                    -o UserKnownHostsFile=/dev/null \
                    ubuntu@${ipAddress} \"echo PubkeyAcceptedKeyTypes +ssh-rsa | sudo tee -a /etc/ssh/sshd_config\"
                """
                sh """ssh -T -i ${SSH_KEY} \
                    -o StrictHostKeyChecking=no \
                    -o UserKnownHostsFile=/dev/null \
                    ubuntu@${ipAddress} \"sudo systemctl restart ssh.service\"
                """
                //////////////////////////////////////////////////////////////////////////////////////////////
            } // stage("Spawn Remote VM")
///////////////////////////////////////////////////////////////////////////////////////
// Checks before installation
///////////////////////////////////////////////////////////////////////////////////////
            stage('Checks before installation') {
                if (!ipAddress?.trim()) {
                    error('Missing VM IP address, cannot run pre-installation checks')
                }

                sshCommand remote: remote, command: 'cloud-init status --wait'
                sshCommand remote: remote, command: 'sudo apt-get -y update'
                sshCommand remote: remote, command: 'sudo apt-get -y install chrony'
                sshCommand remote: remote, command: 'sudo service chrony stop'
                sshCommand remote: remote, command: 'sudo chronyd -vq'
                sshCommand remote: remote, command: 'sudo service chrony start'
            } // stage("Checks before installation")
///////////////////////////////////////////////////////////////////////////////////////
// Install
///////////////////////////////////////////////////////////////////////////////////////
            stage('Install') {
                if (!ipAddress?.trim()) {
                    error('Missing VM IP address, cannot run installation steps')
                }

                sshCommand remote: remote, command: """
                    wget ${INSTALLER_URL}
                    chmod +x ./install_osm.sh
                    sed -i '1 i\\export PATH=/snap/bin:\$PATH' ~/.bashrc
                """


                Map gitlabCredentialsMap = [$class: 'UsernamePasswordMultiBinding',
                                            credentialsId: 'gitlab-registry',
                                            usernameVariable: 'USERNAME',
                                            passwordVariable: 'PASSWORD']
                withCredentials([gitlabCredentialsMap]) {
                    List<String> installArgs = buildInstallerArgs(USERNAME, PASSWORD)

                    String installCmd = "./install_osm.sh ${installArgs.join(' ')}"
                    sshCommand remote: remote, command: """
                        ${installCmd}
                    """
                }
            } // stage("Install")

///////////////////////////////////////////////////////////////////////////////////////
// Health check of installed OSM in remote VM
///////////////////////////////////////////////////////////////////////////////////////
            stage('OSM Health') {
                if (!ipAddress?.trim()) {
                    error('Missing VM IP address, cannot run OSM health checks')
                }
                if (!remote) {
                    error('Missing remote target, cannot run OSM health checks')
                }

                timeout(time: 5, unit: 'MINUTES') {
                    String osmHostname = "nbi.${ipAddress}.nip.io"
                    sshCommand remote: remote, command: """
                        OSM_HOSTNAME=${osmHostname} ~/.local/bin/osm vim-list
                    """
                }
            } // stage('OSM Health')

///////////////////////////////////////////////////////////////////////////////////////
// Get OSM Kubeconfig and store it for future usage (Robot/vCluster)
///////////////////////////////////////////////////////////////////////////////////////
            if (params.DO_ROBOT) {
                stage('OSM Get kubeconfig') {
                    kubeTmpDir = pwd(tmp: true)
                    env.OSM_KUBECONFIG_PATH = "${kubeTmpDir}/osm_config"
                    env.VCLUSTER_KUBECONFIG_PATH = "${kubeTmpDir}/vcluster_config"

                    sshGet remote: remote,
                           from: "/home/ubuntu/.kube/config",
                           into: env.OSM_KUBECONFIG_PATH,
                           override: true
                    sh "chmod 600 ${env.OSM_KUBECONFIG_PATH}"
                    sh "test -s ${env.OSM_KUBECONFIG_PATH}"
                    // Debug: show the Kubernetes API endpoint used by the kubeconfig.
                    // (k3s defaults to 127.0.0.1:6443, which is not reachable from the Jenkins agent container)
                    sh "grep -nE '^\\s*server:' ${env.OSM_KUBECONFIG_PATH} || true"
                } // stage('OSM Get kubeconfig')

///////////////////////////////////////////////////////////////////////////////////////
// Create vCluster for GitOps/Robot test execution
///////////////////////////////////////////////////////////////////////////////////////
                stage('Create vCluster') {
                    println("Creating vcluster ${VCLUSTER_NAME} in namespace ${VCLUSTER_NAMESPACE}")
                    dockerLoginInternalRegistry()
                    create_vcluster(INTERNAL_DOCKER_REGISTRY, testsImageTag, env.OSM_KUBECONFIG_PATH, env.VCLUSTER_KUBECONFIG_PATH, VCLUSTER_NAME, VCLUSTER_NAMESPACE)
                    sh "chmod 600 ${env.VCLUSTER_KUBECONFIG_PATH}"
                    sh "test -s ${env.VCLUSTER_KUBECONFIG_PATH}"
                } // stage('Create vCluster')

///////////////////////////////////////////////////////////////////////////////////////
// Execute Robot tests
///////////////////////////////////////////////////////////////////////////////////////
                stage('System Integration Test') {
                    String prometheusHostname = "prometheus.${ipAddress}.nip.io"
                    Integer prometheusPort = PROMETHEUS_PORT_DEFAULT
                    String osmHostnameRobot = "nbi.${ipAddress}.nip.io:443"

                    register_etsi_vim_account(
                        INTERNAL_DOCKER_REGISTRY,
                        testsImageTag,
                        osmHostnameRobot,
                        params.ROBOT_VIM,
                        params.ROBOT_PORT_MAPPING_VIM,
                        params.KUBECONFIG,
                        params.CLOUDS,
                        params.PROMETHEUS_CONFIG_VIM
                    )
                    register_etsi_k8s_cluster(
                        INTERNAL_DOCKER_REGISTRY,
                        testsImageTag,
                        osmHostnameRobot,
                        params.ROBOT_VIM,
                        params.ROBOT_PORT_MAPPING_VIM,
                        params.KUBECONFIG,
                        params.CLOUDS,
                        params.PROMETHEUS_CONFIG_VIM
                    )

                    // IMPORTANT: tests expect the vcluster kubeconfig at this container path.
                    String robotVclusterKubeconfigPath = ROBOT_VCLUSTER_KUBECONFIG_CONTAINER_PATH
                    run_robot_systest(
                        INTERNAL_DOCKER_REGISTRY,
                        testsImageTag,
                        params.ROBOT_TAG_NAME,
                        osmHostnameRobot,
                        prometheusHostname,
                        prometheusPort,
                        params.OCI_REGISTRY_URL,
                        params.ROBOT_VIM,
                        params.ROBOT_PORT_MAPPING_VIM,
                        params.KUBECONFIG,
                        params.CLOUDS,
                        null,
                        SSH_KEY,
                        params.ROBOT_PASS_THRESHOLD,
                        params.ROBOT_UNSTABLE_THRESHOLD,
                        // extraEnvVars map of extra environment variables
                        ['CLUSTER_KUBECONFIG_CREDENTIALS': robotVclusterKubeconfigPath],
                        // extraVolMounts map of extra volume mounts
                        [(env.VCLUSTER_KUBECONFIG_PATH): robotVclusterKubeconfigPath]
                    )
                } // stage('System Integration Test')
            } else {
                println('Skipping kubeconfig/vcluster steps because DO_ROBOT is set to false')
            }
        } else {
            println('Skipping VM spawn because DO_INSTALL is set to false')
        }
    } finally {
        stage('Archive Logs') {
            if (params.DO_INSTALL && remote) {
                try {
                    archiveLogs(remote)
                } catch (Exception e) {
                    println("Archive logs failed: ${e.message}")
                }
            } else {
                println('No remote target to collect logs from')
            }
        }

        stage('Cleanup') {
            // Always attempt to cleanup temp kubeconfig directory if created.
            if (kubeTmpDir?.trim()) {
                sh "rm -rf ${kubeTmpDir} || true"
                kubeTmpDir = null
            }

            if (!params.DO_INSTALL || serverId == null) {
                println('No VM to cleanup')
                return
            }

            String buildState = currentBuild.currentResult ?: 'SUCCESS'
            boolean buildFailed = buildState == 'FAILURE'

            boolean deleteVm = true
            if (buildFailed && params.SAVE_CONTAINER_ON_FAIL) {
                deleteVm = false
            }
            if (!buildFailed && params.SAVE_CONTAINER_ON_PASS) {
                deleteVm = false
            }

            if (deleteVm) {
                println("Deleting VM: ${serverId}")
                try {
                    runHiveCommand("""
                        openstack server delete ${serverId}
                    """)
                } catch (Exception e) {
                    // Avoid masking an earlier failure with cleanup failure.
                    println("VM delete failed: ${e.message}")
                }
            } else {
                println("Preserving VM ${serverId} (build state: ${buildState})")
            }
        }
    }
}

////////////////////////////////////////////////////////////////////////////////////////
// Helper Classes & Functions (ported from ci_stage_3.groovy)
// Placed below the pipeline for readability.
////////////////////////////////////////////////////////////////////////////////////////

/** Usage:
 *   def dr = new DockerRunner(this)
 *   stdout = dr.run(
 *       image   : "${INTERNAL_DOCKER_REGISTRY}opensourcemano/tests:${tag}",
 *       entry   : "osm",        // optional
 *       envVars : [ "OSM_HOSTNAME=${host}" ],
 *       envFile : myEnv,
 *       mounts  : [
 *                  "${clouds}:/etc/openstack/clouds.yaml",
 *                  "${kubeconfig}:/root/.kube/config"
 *                ],
 *       cmd     : "vim-create --name osm …"
 *   )
 */
class DockerRunner implements Serializable {
    def steps
    DockerRunner(def steps) { this.steps = steps }

    /** Returns stdout (trimmed) if returnStdout is true; throws Exception on non-zero exit */
    String run(Map args = [:]) {
        def returnStdout = args.remove('returnStdout') ?: false
        def envFile = args.envFile ?: ''
        def entry = args.entry ? "--entrypoint ${args.entry}" : ''
        def mounts = (args.mounts ?: [])
            .findAll { it && it.trim() }
            .collect { "-v ${it}" }
            .join(' ')
        def envs = (args.envVars ?: [])
            .findAll { it && it.trim() }
            .collect { "--env ${it}" }
            .join(' ')
        def image = args.image ?: ''
        def cmd = args.cmd ?: ''
        def fullCmd = "docker run --rm ${entry} ${envs} ${envFile ? "--env-file ${envFile}" : ''} ${mounts} ${image} ${cmd}".trim()

        def result = null
        try {
            if (returnStdout) {
                result = steps.sh(returnStdout: true, script: fullCmd).trim()
            } else {
                steps.sh(script: fullCmd)
            }
        } catch (Exception ex) {
            throw new Exception("docker run failed -> ${ex.message}")
        } finally {
            steps.echo("Command executed: ${fullCmd}")
        }
        return result ?: ''
    }
}

/* -------------------------------------------------------------------
 *  create_vcluster  – spin up a vcluster in the target OSM cluster
 * @params:
 *  tagName - The OSM test docker image tag to use
 *  kubeconfigPath - The path of the OSM kubernetes master configuration file
 *  vclusterKubeconfigOutPath - Output path for the vcluster kubeconfig
 *  vclusterName - Name of the vcluster
 *  vclusterNamespace - Namespace for the vcluster
 * ------------------------------------------------------------------- */
void create_vcluster(String dockerRegistryUrl, String tagName, String kubeconfigPath, String vclusterKubeconfigOutPath, String vclusterName, String vclusterNamespace) {
    def dr = new DockerRunner(this)
    def mounts = ["${kubeconfigPath}:/root/.kube/config"]
    def envs = ["KUBECONFIG=/root/.kube/config"]
    def image = "${dockerRegistryUrl}opensourcemano/tests:${tagName}"

    // 1) create vcluster namespace
    println("vcluster: ensuring namespace '${vclusterNamespace}' exists")
    dr.run(
        image: image,
        entry: 'kubectl',
        envVars: envs,
        mounts: mounts,
        cmd: "create namespace ${vclusterNamespace} || true"
    )

    // 2) create vcluster (no connect)
    println("vcluster: creating '${vclusterName}' (no connect)")
    dr.run(
        image: image,
        entry: 'vcluster',
        envVars: envs,
        mounts: mounts,
        cmd: "create ${vclusterName} -n ${vclusterNamespace} --connect=false -f /etc/vcluster.yaml"
    )

    // 3) poll until Status is Running
    int maxWaitMinutes = 10
    long deadline = System.currentTimeMillis() + (maxWaitMinutes * 60 * 1000)
    boolean running = false
    String lastOut = ''

    println("vcluster: waiting for '${vclusterName}' to reach status 'Running' (timeout: ${maxWaitMinutes} minutes)")
    while (System.currentTimeMillis() < deadline) {
        try {
            lastOut = dr.run(
                returnStdout: true,
                image: image,
                entry: '/bin/sh',
                envVars: envs,
                mounts: mounts,
                cmd: """-c 'vcluster list --output json | jq -r ".[] | select(.Name==\\"${vclusterName}\\") | .Status"'"""
            ).trim()
        } catch (Exception e) {
            println("Polling command failed: ${e.message}. Will retry.")
            lastOut = "Error: ${e.message}"
        }

        println("Polling for vcluster status. Current status: '${lastOut}'")
        if (lastOut == 'Running') {
            running = true
            break
        }
        sleep 10
    }

    if (!running) {
        println("vcluster status after timeout: ${lastOut}")
        throw new Exception("vcluster '${vclusterName}' did not reach 'Running' state within ${maxWaitMinutes} minutes.")
    }

    // 4) get vcluster kubeconfig
    String outPath = vclusterKubeconfigOutPath ?: "${WORKSPACE}/kubeconfig/vcluster_config"
    // Ensure destination directory exists on the Jenkins agent before relying on shell redirection.
    String outDir = outPath.contains('/') ? outPath.substring(0, outPath.lastIndexOf('/')) : '.'
    sh "mkdir -p ${outDir}"
    println("vcluster: exporting kubeconfig to '${outPath}'")
    dr.run(
        image: image,
        entry: 'vcluster',
        envVars: envs,
        mounts: mounts,
        cmd: "connect ${vclusterName} -n ${vclusterNamespace} --server ${vclusterName}.${vclusterNamespace}.svc.cluster.local:443 --print > ${outPath}"
    )
}

void retryWithDocker(int maxAttempts, int delaySeconds, Closure action) {
    int attempts = maxAttempts
    while (attempts >= 0) {
        try {
            if (action()) return
        } catch (Exception e) {
            println("Attempt failed: ${e.message}")
        }
        println("Retrying... (${attempts} attempts left)")
        sleep delaySeconds
        attempts--
    }
    throw new Exception("Operation failed after ${maxAttempts} retries")
}

void register_etsi_vim_account(
    String dockerRegistryUrl,
    String tagName,
    String osmHostname,
    String envfile = null,
    String portmappingfile = null,
    String kubeconfig = null,
    String clouds = null,
    String prometheusconfigfile = null
) {
    String VIM_TARGET = "osm"
    String VIM_MGMT_NET = "osm-ext"
    String OS_PROJECT_NAME = "osm_jenkins"
    String OS_AUTH_URL = "http://172.21.247.1:5000/v3"
    String entrypointCmd = "osm"

    def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim()
    String environmentFile = envfile ?: "${tempdir}/env"
    if (!envfile) {
        sh(script: "touch ${environmentFile}")
    }

    retryWithDocker(3, 10) {
        def dr = new DockerRunner(this)
        try {
            println("Attempting to register VIM account")
            withCredentials([usernamePassword(credentialsId: 'openstack-jenkins-credentials',
                passwordVariable: 'OS_PASSWORD', usernameVariable: 'OS_USERNAME')]) {
                String entrypointArgs = """vim-create --name ${VIM_TARGET} --user ${OS_USERNAME} \
                        --password ${OS_PASSWORD} --tenant ${OS_PROJECT_NAME} \
                        --auth_url ${OS_AUTH_URL} --account_type openstack --description vim \
                        --prometheus_config_file /root/etsi-vim-prometheus.json \
                        --config '{management_network_name: ${VIM_MGMT_NET}, dataplane_physical_net: physnet2}' || true"""
                dr.run(
                    image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
                    entry: entrypointCmd,
                    envVars: ["OSM_HOSTNAME=${osmHostname}"],
                    envFile: environmentFile,
                    mounts: [
                        clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                        kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                        portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                        prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
                    ].findAll { it != null },
                    cmd: entrypointArgs,
                    returnStdout: true
                )
            }

            // Check if the VIM is ENABLED
            int statusChecks = 5
            while (statusChecks > 0) {
                sleep 10
                String vimList = dr.run(
                    image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
                    entry: entrypointCmd,
                    envVars: ["OSM_HOSTNAME=${osmHostname}"],
                    envFile: environmentFile,
                    mounts: [
                        clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                        kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                        portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                        prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
                    ].findAll { it != null },
                    cmd: "vim-list --long | grep ${VIM_TARGET}",
                    returnStdout: true
                )
                if (vimList.contains("ENABLED")) {
                    println("VIM successfully registered and is ENABLED.")
                    return true
                }
                statusChecks--
            }
        } catch (Exception e) {
            println("VIM registration check failed: ${e.message}")
        }

        // If we get here, VIM is not enabled or creation failed. cleanup and retry.
        println("VIM not enabled, deleting and retrying...")
        dr.run(
            image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
            entry: entrypointCmd,
            envVars: ["OSM_HOSTNAME=${osmHostname}"],
            envFile: environmentFile,
            mounts: [
                clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
            ].findAll { it != null },
            cmd: "vim-delete --force ${VIM_TARGET}",
            returnStdout: true
        )
        return false
    }
}

void register_etsi_k8s_cluster(
    String dockerRegistryUrl,
    String tagName,
    String osmHostname,
    String envfile = null,
    String portmappingfile = null,
    String kubeconfig = null,
    String clouds = null,
    String prometheusconfigfile = null
) {
    String K8S_CLUSTER_TARGET = "osm"
    String VIM_TARGET = "osm"
    String VIM_MGMT_NET = "osm-ext"
    String K8S_CREDENTIALS = "/root/.kube/config"
    String entrypointCmd = "osm"

    def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim()
    String environmentFile = envfile ?: "${tempdir}/env"
    if (!envfile) {
        sh(script: "touch ${environmentFile}")
    }

    retryWithDocker(3, 10) {
        def dr = new DockerRunner(this)
        try {
            println("Attempting to register K8s cluster")
            dr.run(
                image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
                entry: entrypointCmd,
                envVars: ["OSM_HOSTNAME=${osmHostname}"],
                envFile: environmentFile,
                mounts: [
                    clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                    kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                    portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                    prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
                ].findAll { it != null },
                cmd: """k8scluster-add ${K8S_CLUSTER_TARGET} --creds ${K8S_CREDENTIALS} --version \"v1\" \
                        --description \"Robot-cluster\" --skip-jujubundle --vim ${VIM_TARGET} \
                        --k8s-nets '{net1: ${VIM_MGMT_NET}}'""",
                returnStdout: true
            )

            // Check if the K8s cluster is ENABLED
            int statusChecks = 10
            while (statusChecks > 0) {
                sleep 10
                String clusterList = dr.run(
                    image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
                    entry: entrypointCmd,
                    envVars: ["OSM_HOSTNAME=${osmHostname}"],
                    envFile: environmentFile,
                    mounts: [
                        clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                        kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                        portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                        prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
                    ].findAll { it != null },
                    cmd: "k8scluster-list | grep ${K8S_CLUSTER_TARGET}",
                    returnStdout: true
                )
                if (clusterList.contains("ENABLED")) {
                    println("K8s cluster successfully registered and is ENABLED.")
                    return true
                }
                statusChecks--
            }
        } catch (Exception e) {
            println("K8s cluster registration check failed: ${e.message}")
        }

        // If we get here, cluster is not enabled or creation failed. cleanup and retry.
        println("K8s cluster not enabled, deleting and retrying...")
        dr.run(
            image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
            entry: entrypointCmd,
            envVars: ["OSM_HOSTNAME=${osmHostname}"],
            envFile: environmentFile,
            mounts: [
                clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                prometheusconfigfile ? "${prometheusconfigfile}:/root/etsi-vim-prometheus.json" : null
            ].findAll { it != null },
            cmd: "k8scluster-delete ${K8S_CLUSTER_TARGET}"
        )
        return false
    }
}

void run_robot_systest(
    String dockerRegistryUrl,
    String tagName,
    String testName,
    String osmHostname,
    String prometheusHostname,
    Integer prometheusPort = null,
    String ociRegistryUrl = null,
    String envfile = null,
    String portmappingfile = null,
    String kubeconfig = null,
    String clouds = null,
    String hostfile = null,
    String osmRSAfile = null,
    String passThreshold = '0.0',
    String unstableThreshold = '0.0',
    Map extraEnvVars = null,
    Map extraVolMounts = null
) {
    def tempdir = sh(returnStdout: true, script: 'mktemp -d').trim()
    String environmentFile = envfile ?: "${tempdir}/env"
    if (!envfile) {
        sh(script: "touch ${environmentFile}")
    }

    def prometheusPortVar = prometheusPort != null ? "PROMETHEUS_PORT=${prometheusPort}" : null
    def hostfilemount = hostfile ? "${hostfile}:/etc/hosts" : null

    try {
        withCredentials([usernamePassword(credentialsId: 'gitlab-oci-test',
            passwordVariable: 'OCI_REGISTRY_PSW', usernameVariable: 'OCI_REGISTRY_USR')]) {
            def baseEnvVars = [
                "OSM_HOSTNAME=${osmHostname}",
                "PROMETHEUS_HOSTNAME=${prometheusHostname}",
                prometheusPortVar,
                ociRegistryUrl ? "OCI_REGISTRY_URL=${ociRegistryUrl}" : null,
                "OCI_REGISTRY_USER=${OCI_REGISTRY_USR}",
                "OCI_REGISTRY_PASSWORD=${OCI_REGISTRY_PSW}"
            ].findAll { it != null }

            def baseMounts = [
                clouds ? "${clouds}:/etc/openstack/clouds.yaml" : null,
                osmRSAfile ? "${osmRSAfile}:/root/osm_id_rsa" : null,
                kubeconfig ? "${kubeconfig}:/root/.kube/config" : null,
                "${tempdir}:/robot-systest/reports",
                portmappingfile ? "${portmappingfile}:/root/port-mapping.yaml" : null,
                hostfilemount
            ].findAll { it != null }

            def extraEnvVarsList = extraEnvVars?.collect { key, value -> "${key}=${value}" } ?: []
            def extraVolMountsList = extraVolMounts?.collect { hostPath, containerPath -> "${hostPath}:${containerPath}" } ?: []

            def dr = new DockerRunner(this)
            dr.run(
                image: "${dockerRegistryUrl}opensourcemano/tests:${tagName}",
                envVars: baseEnvVars + extraEnvVarsList,
                envFile: "${environmentFile}",
                mounts: baseMounts + extraVolMountsList,
                cmd: "-t ${testName}"
            )
        }
    } finally {
        // Best-effort publish Robot results from tempdir into workspace
        sh("cp ${tempdir}/*.xml . 2>/dev/null || true")
        sh("cp ${tempdir}/*.html . 2>/dev/null || true")

        def outputDirectory = sh(returnStdout: true, script: 'pwd').trim()
        sh("command -v tree >/dev/null 2>&1 && tree ${outputDirectory} || ls -la ${outputDirectory}")

        try {
            step([
                $class: 'RobotPublisher',
                outputPath: "${outputDirectory}",
                outputFileName: '*.xml',
                disableArchiveOutput: false,
                reportFileName: 'report.html',
                logFileName: 'log.html',
                passThreshold: passThreshold,
                unstableThreshold: unstableThreshold,
                otherFiles: '*.png'
            ])
        } catch (Exception e) {
            println("RobotPublisher failed: ${e.message}")
        }
    }
}

String get_value(String key, String output) {
    for (String line : output.split('\n')) {
        def data = line.split('\\|')
        if (data.length > 1 && data[1].trim() == key) {
            return data[2].trim()
        }
    }
    return null
}

void dockerLoginInternalRegistry() {
    withCredentials([usernamePassword(credentialsId: 'gitlab-registry',
        passwordVariable: 'REGISTRY_PASSWORD', usernameVariable: 'REGISTRY_USERNAME')]) {
        sh """
            set -e
            echo "${REGISTRY_PASSWORD}" | docker login ${INTERNAL_DOCKER_REGISTRY_HOST} -u "${REGISTRY_USERNAME}" --password-stdin
        """
    }
}

String withHiveEnv(String commandBody) {
    """#!/bin/sh -e
${HIVE_ENV_EXPORT}
${commandBody.stripIndent()}
"""
}

String runHiveCommand(String commandBody) {
    sh(returnStdout: true, script: withHiveEnv(commandBody)).trim()
}

String waitForServerIp(String id) {
    String addr = ''
    timeout(time: 5, unit: 'MINUTES') {
        waitUntil {
            def showOutput = runHiveCommand("""
                openstack server show ${id}
            """)
            def rawAddress = get_value('addresses', showOutput)
            if (rawAddress) {
                addr = rawAddress.split('=')[1]
                return true
            }
            sleep 5
            return false
        }
    }
    return addr
}

// Collect logs from the remote VM and archive them in Jenkins
void archiveLogs(Map remoteTarget) {
    sshCommand remote: remoteTarget, command: '''mkdir -p logs/dags logs/vcluster logs/flux-system logs/events logs/system'''

    // Collect Kubernetes events
    sshCommand remote: remoteTarget, command: '''
        echo "Extracting Kubernetes events"
        kubectl get events --all-namespaces --sort-by='.lastTimestamp' -o wide > logs/events/k8s-events.log 2>&1 || true
        kubectl get events -n osm --sort-by='.lastTimestamp' -o wide > logs/events/osm-events.log 2>&1 || true
        kubectl get events -n vcluster --sort-by='.lastTimestamp' -o wide > logs/events/vcluster-events.log 2>&1 || true
        kubectl get events -n flux-system --sort-by='.lastTimestamp' -o wide > logs/events/flux-system-events.log 2>&1 || true
    '''

    // Collect host logs and system info
    sshCommand remote: remoteTarget, command: '''
      echo "Collect system logs"
      if command -v journalctl >/dev/null; then
        journalctl > logs/system/system.log
      fi

      for entry in syslog messages; do
        [ -e "/var/log/${entry}" ] && cp -f /var/log/${entry} logs/system/"${entry}.log"
      done

      echo "Collect active services"
      case "$(cat /proc/1/comm)" in
        systemd)
          systemctl list-units > logs/system/services.txt 2>&1
          ;;
        *)
          service --status-all >> logs/system/services.txt 2>&1
          ;;
      esac

      top -b -n 1 > logs/system/top.txt 2>&1
      ps fauxwww > logs/system/ps.txt 2>&1
    '''

    // Collect OSM namespace workloads
    sshCommand remote: remoteTarget, command: '''
        for deployment in `kubectl -n osm get deployments | grep -v operator | grep -v NAME| awk '{print $1}'`; do
            echo "Extracting log for $deployment"
            kubectl -n osm logs deployments/$deployment --timestamps=true --all-containers 2>&1 > logs/$deployment.log || true
        done
    '''
    sshCommand remote: remoteTarget, command: '''
        for statefulset in `kubectl -n osm get statefulsets | grep -v operator | grep -v NAME| awk '{print $1}'`; do
            echo "Extracting log for $statefulset"
            kubectl -n osm logs statefulsets/$statefulset --timestamps=true --all-containers 2>&1 > logs/$statefulset.log || true
        done
    '''
    sshCommand remote: remoteTarget, command: '''
        schedulerPod="$(kubectl get pods -n osm | grep osm-scheduler| awk '{print $1; exit}')"; \
        echo "Extracting logs from Airflow DAGs from pod ${schedulerPod}"; \
        kubectl -n osm cp ${schedulerPod}:/opt/airflow/logs/scheduler/latest/dags logs/dags -c scheduler 2>&1 || true
    '''

    // Collect vcluster and flux-system namespace logs
    sshCommand remote: remoteTarget, command: '''
        echo "Extracting logs from vcluster namespace"
        for pod in `kubectl get pods -n vcluster | grep -v NAME | awk '{print $1}'`; do
            echo "Extracting log for vcluster pod: $pod"
            kubectl logs -n vcluster $pod --timestamps=true --all-containers 2>&1 > logs/vcluster/$pod.log || true
        done
    '''
    sshCommand remote: remoteTarget, command: '''
        echo "Extracting logs from flux-system namespace"
        for pod in `kubectl get pods -n flux-system | grep -v NAME | awk '{print $1}'`; do
            echo "Extracting log for flux-system pod: $pod"
            kubectl logs -n flux-system $pod --timestamps=true --all-containers 2>&1 > logs/flux-system/$pod.log || true
        done
    '''

    sh 'rm -rf logs'
    sshCommand remote: remoteTarget, command: '''ls -al logs logs/vcluster logs/events logs/flux-system logs/system'''
    sshGet remote: remoteTarget, from: 'logs', into: '.', override: true
    archiveArtifacts artifacts: 'logs/*.log, logs/dags/*.log, logs/vcluster/*.log, logs/events/*.log, logs/flux-system/*.log, logs/system/**'
}