X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=jenkins%2Fci-pipelines%2Fci_stage_3.groovy;h=05d795dd75bca72d09e58b7efe7d568c77f322d5;hb=9f97500bc31e9a973375fa43ac802f187b9f5740;hp=954c692d39398373d6b46573d89837cf7856f49d;hpb=44d3d012155f42e2e0a2301cc7cfb1b6f68ca452;p=osm%2Fdevops.git diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 954c692d..05d795dd 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -32,22 +32,24 @@ properties([ string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), string(defaultValue: 'osm-stage_4', description: '', name: 'DOWNSTREAM_STAGE_NAME'), string(defaultValue: 'testing-daily', description: '', name: 'DOCKER_TAG'), + string(defaultValue: 'ubuntu20.04', description: '', name: 'OPENSTACK_BASE_IMAGE'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_FAIL'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'), booleanParam(defaultValue: true, description: '', name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS'), - booleanParam(defaultValue: true, description: '', name: 'DO_STAGE_4'), booleanParam(defaultValue: true, description: '', name: 'DO_BUILD'), booleanParam(defaultValue: true, description: '', name: 'DO_INSTALL'), - booleanParam(defaultValue: true, description: '', name: 'DO_SMOKE'), booleanParam(defaultValue: true, description: '', name: 'DO_DOCKERPUSH'), booleanParam(defaultValue: false, description: '', name: 'SAVE_ARTIFACTS_OVERRIDE'), string(defaultValue: '/home/jenkins/hive/openstack-etsi.rc', description: '', name: 'HIVE_VIM_1'), - booleanParam(defaultValue: false, description: '', name: 'DO_ROBOT'), - string(defaultValue: 'sanity', description: 'sanity/regression are the options', name: 'TEST_NAME'), + booleanParam(defaultValue: true, description: '', name: 'DO_ROBOT'), + string(defaultValue: 'sanity', description: 'sanity/regression/daily are the common options', name: 'ROBOT_TAG_NAME'), string(defaultValue: '/home/jenkins/hive/robot-systest.cfg', description: '', name: 'ROBOT_VIM'), + string(defaultValue: '/home/jenkins/hive/port-mapping-etsi-vim.yaml', description: 'Port mapping file for SDN assist in ETSI VIM', name: 'ROBOT_PORT_MAPPING_VIM'), string(defaultValue: '/home/jenkins/hive/kubeconfig.yaml', description: '', name: 'KUBECONFIG'), string(defaultValue: '/home/jenkins/hive/clouds.yaml', description: '', name: 'CLOUDS'), string(defaultValue: 'Default', description: '', name: 'INSTALLER'), + string(defaultValue: '100.0', description: '% passed Robot tests to mark the build as passed', name: 'ROBOT_PASS_THRESHOLD'), + string(defaultValue: '80.0', description: '% passed Robot tests to mark the build as unstable (if lower, it will be failed)', name: 'ROBOT_UNSTABLE_THRESHOLD'), ]) ]) @@ -55,10 +57,22 @@ properties([ //////////////////////////////////////////////////////////////////////////////////////// // Helper Functions //////////////////////////////////////////////////////////////////////////////////////// -def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus_port=null,envfile=null,kubeconfig=null,clouds=null,hostfile=null,jujuPassword=null) { - tempdir = sh(returnStdout: true, script: "mktemp -d").trim() - if ( !envfile ) - { +void run_robot_systest(String tagName, + String testName, + String osmHostname, + String prometheusHostname, + Integer prometheusPort=null, + String envfile=null, + String portmappingfile=null, + String kubeconfig=null, + String clouds=null, + String hostfile=null, + String jujuPassword=null, + String osmRSAfile=null, + String pass_th='0.0', + String unstable_th='0.0') { + tempdir = sh(returnStdout: true, script: 'mktemp -d').trim() + if ( !envfile ) { sh(script: "touch ${tempdir}/env") envfile="${tempdir}/env" } @@ -77,7 +91,7 @@ def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus } try { - sh "docker run --env OSM_HOSTNAME=${osmHostname} --env PROMETHEUS_HOSTNAME=${prometheusHostname} ${PROMETHEUS_PORT_VAR} ${JUJU_PASSWORD_VAR} --env-file ${envfile} -v ${clouds}:/etc/openstack/clouds.yaml -v ${kubeconfig}:/root/.kube/config -v ${tempdir}:/robot-systest/reports ${hostfilemount} opensourcemano/tests:${tagName} -c -t ${testName}" + sh "docker run --env OSM_HOSTNAME=${osmHostname} --env PROMETHEUS_HOSTNAME=${prometheusHostname} ${PROMETHEUS_PORT_VAR} ${JUJU_PASSWORD_VAR} --env-file ${envfile} -v ${clouds}:/etc/openstack/clouds.yaml -v ${osmRSAfile}:/root/osm_id_rsa -v ${kubeconfig}:/root/.kube/config -v ${tempdir}:/robot-systest/reports -v ${portmappingfile}:/root/port-mapping.yaml ${hostfilemount} opensourcemano/tests:${tagName} -c -t ${testName}" } finally { sh "cp ${tempdir}/* ." outputDirectory = sh(returnStdout: true, script: "pwd").trim() @@ -89,8 +103,8 @@ def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus disableArchiveOutput : false, reportFileName : "report.html", logFileName : "log.html", - passThreshold : 0, - unstableThreshold: 0, + passThreshold : pass_th, + unstableThreshold: unstable_th, otherFiles : "*.png", ]) } @@ -186,7 +200,9 @@ node("${params.NODE}") { /////////////////////////////////////////////////////////////////////////////////////// stage("Copy Artifacts") { // cleanup any previous repo - sh 'rm -rf repo' + sh "tree -fD repo || exit 0" + sh 'rm -rvf repo' + sh "tree -fD repo && lsof repo || exit 0" dir("repo") { packageList = [] dir("${RELEASE}") { @@ -198,32 +214,33 @@ node("${params.NODE}") { def upstreamComponent="" if ( params.UPSTREAM_JOB_NAME ) { println("Fetching upstream job artifact from ${params.UPSTREAM_JOB_NAME}") + lock('Artifactory') { + step ([$class: 'CopyArtifact', + projectName: "${params.UPSTREAM_JOB_NAME}", + selector: [$class: 'SpecificBuildSelector', + buildNumber: "${params.UPSTREAM_JOB_NUMBER}"] + ]) + + upstreamComponent = ci_helper.get_mdg_from_project( + ci_helper.get_env_value('build.env','GERRIT_PROJECT')) + def buildNumber = ci_helper.get_env_value('build.env','BUILD_NUMBER') + dir("$upstreamComponent") { + // the upstream job name contains suffix with the project. Need this stripped off + def project_without_branch = params.UPSTREAM_JOB_NAME.split('/')[0] + def packages = ci_helper.get_archive(params.ARTIFACTORY_SERVER, + upstreamComponent, + GERRIT_BRANCH, + "${project_without_branch} :: ${GERRIT_BRANCH}", + buildNumber) - step ([$class: 'CopyArtifact', - projectName: "${params.UPSTREAM_JOB_NAME}", - selector: [$class: 'SpecificBuildSelector', - buildNumber: "${params.UPSTREAM_JOB_NUMBER}"] - ]) - - upstreamComponent = ci_helper.get_mdg_from_project( - ci_helper.get_env_value('build.env','GERRIT_PROJECT')) - def buildNumber = ci_helper.get_env_value('build.env','BUILD_NUMBER') - dir("$upstreamComponent") { - // the upstream job name contains suffix with the project. Need this stripped off - def project_without_branch = params.UPSTREAM_JOB_NAME.split('/')[0] - def packages = ci_helper.get_archive(params.ARTIFACTORY_SERVER, - upstreamComponent, - GERRIT_BRANCH, - "${project_without_branch} :: ${GERRIT_BRANCH}", - buildNumber) - - packageList.addAll(packages) - println("Fetched pre-merge ${params.UPSTREAM_JOB_NAME}: ${packages}") - } + packageList.addAll(packages) + println("Fetched pre-merge ${params.UPSTREAM_JOB_NAME}: ${packages}") + } + } // lock artifactory } parallelSteps = [:] - def list = ["RO", "openvim", "osmclient", "IM", "devops", "MON", "N2VC", "NBI", "common", "LCM", "POL", "LW-UI", "NG-UI", "PLA", "tests"] + def list = ["RO", "osmclient", "IM", "devops", "MON", "N2VC", "NBI", "common", "LCM", "POL", "NG-UI", "PLA", "tests"] if (upstreamComponent.length()>0) { println("Skipping upstream fetch of "+upstreamComponent) list.remove(upstreamComponent) @@ -248,16 +265,22 @@ node("${params.NODE}") { } } } - parallel parallelSteps + lock('Artifactory') { + parallel parallelSteps + } /////////////////////////////////////////////////////////////////////////////////////// // Create Devops APT repository /////////////////////////////////////////////////////////////////////////////////////// - sh "mv devops/pool/ pool" - sh "dpkg-sig --sign builder -k ${GPG_KEY_NAME} pool/devops/*" - sh "mkdir -p dists/${params.REPO_DISTRO}/devops/binary-amd64/" - sh "apt-ftparchive packages pool/devops > dists/${params.REPO_DISTRO}/devops/binary-amd64/Packages" - sh "gzip -9fk dists/${params.REPO_DISTRO}/devops/binary-amd64/Packages" + sh "mkdir -p pool" + for (component in [ "devops", "IM", "osmclient" ]) { + sh "ls -al ${component}/pool/" + sh "cp -r ${component}/pool/* pool/" + sh "dpkg-sig --sign builder -k ${GPG_KEY_NAME} pool/${component}/*" + sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/" + sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + } // create and sign the release file sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release" @@ -273,12 +296,17 @@ node("${params.NODE}") { http_server_name = "${container_name}-apache" pwd = sh(returnStdout:true, script: 'pwd').trim() - repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim() - repo_base_url = ci_helper.start_http_server(pwd,http_server_name,repo_port) - NODE_IP_ADDRESS=sh(returnStdout: true, script: + repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0));' + + 'print(s.getsockname()[1]); s.close()\');', + returnStdout: true).trim() + internal_docker_http_server_url = ci_helper.start_http_server(pwd, http_server_name, repo_port) + NODE_IP_ADDRESS = sh(returnStdout: true, script: "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim() + ci_helper.check_status_http_server(NODE_IP_ADDRESS, repo_port) } + sh "tree -fD repo" + // Unpack devops package into temporary location so that we use it from upstream if it was part of a patch osm_devops_dpkg = sh(returnStdout: true, script: "find ./repo/release/pool/ -name osm-devops*.deb").trim() devopstempdir = sh(returnStdout: true, script: "mktemp -d").trim() @@ -343,7 +371,7 @@ node("${params.NODE}") { output=sh(returnStdout: true, script: """#!/bin/sh -e for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done openstack server create --flavor osm.sanity \ - --image ubuntu18.04 \ + --image ${OPENSTACK_BASE_IMAGE} \ --key-name CICD \ --property build_url="${BUILD_URL}" \ --nic net-id=osm-ext \ @@ -372,10 +400,16 @@ node("${params.NODE}") { println("Waiting for VM at ${IP_ADDRESS} to be reachable") alive = false - while (! alive) { - output=sh(returnStdout: true, script: "sleep 1 ; nc -zv ${IP_ADDRESS} 22 2>&1 || true").trim() - println("output is [$output]") - alive = output.contains("succeeded") + timeout(time: 1, unit: 'MINUTES') { + while (!alive) { + output = sh( + returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "-o ConnectTimeout=5 ubuntu@${IP_ADDRESS} 'echo Alive'") + alive = (output == 0) + } } println("VM is ready and accepting ssh connections") } // stage("Spawn Remote VM") @@ -426,8 +460,20 @@ node("${params.NODE}") { remote.logLevel = 'INFO' remote.pty = true + // Ensure the VM is ready + sshCommand remote: remote, command: 'cloud-init status --wait' + + // Force time sync to avoid clock drift and invalid certificates sshCommand remote: remote, command: """ - wget https://osm-download.etsi.org/ftp/osm-9.0-nine/install_osm.sh + sudo apt update + sudo apt install -y chrony + sudo service chrony stop + sudo chrony -vq + sudo service chrony start + """ + + sshCommand remote: remote, command: """ + wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh chmod +x ./install_osm.sh sed -i '1 i\\export PATH=/snap/bin:\${PATH}' ~/.bashrc """ @@ -453,9 +499,9 @@ node("${params.NODE}") { --tag ${container_name} """ } - prometheusHostname = "prometheus."+IP_ADDRESS+".xip.io" + prometheusHostname = "prometheus."+IP_ADDRESS+".nip.io" prometheusPort = 80 - osmHostname = "nbi."+IP_ADDRESS+".xip.io:443" + osmHostname = "nbi."+IP_ADDRESS+".nip.io:443" } else { // Run -k8s installer here specifying internal docker registry and docker proxy withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry', @@ -467,7 +513,8 @@ node("${params.NODE}") { ${release} -r unstable \ -d ${USERNAME}:${PASSWORD}@${INTERNAL_DOCKER_REGISTRY} \ -p ${INTERNAL_DOCKER_PROXY} \ - -t ${container_name} + -t ${container_name} \ + --nocachelxdimages """ } prometheusHostname = IP_ADDRESS @@ -475,62 +522,63 @@ node("${params.NODE}") { osmHostname = IP_ADDRESS } } // stage("Install") - } // if ( params.DO_INSTALL ) - /////////////////////////////////////////////////////////////////////////////////////// // Health check of installed OSM in remote vm /////////////////////////////////////////////////////////////////////////////////////// - if ( params.DO_SMOKE ) { stage("OSM Health") { stackName = "osm" sshCommand remote: remote, command: """ /usr/share/osm-devops/installers/osm_health.sh -k -s ${stackName} """ - } - } + } // stage("OSM Health") + } // if ( params.DO_INSTALL ) + /////////////////////////////////////////////////////////////////////////////////////// // Execute Robot tests /////////////////////////////////////////////////////////////////////////////////////// stage_archive = false - if ( params.DO_STAGE_4 ) { + if ( params.DO_ROBOT ) { try { stage("System Integration Test") { - if ( params.DO_ROBOT ) { - if( useCharmedInstaller ) { - tempdir = sh(returnStdout: true, script: "mktemp -d").trim() - sh(script: "touch ${tempdir}/hosts") - hostfile="${tempdir}/hosts" - sh """cat << EOF > ${hostfile} + if ( useCharmedInstaller ) { + tempdir = sh(returnStdout: true, script: "mktemp -d").trim() + sh(script: "touch ${tempdir}/hosts") + hostfile="${tempdir}/hosts" + sh """cat << EOF > ${hostfile} 127.0.0.1 localhost -${remote.host} prometheus.${remote.host}.xip.io nbi.${remote.host}.xip.io +${remote.host} prometheus.${remote.host}.nip.io nbi.${remote.host}.nip.io EOF""" - } else { - hostfile=null - } - - jujuPassword=sshCommand remote: remote, command: """ - echo `juju gui 2>&1 | grep password | cut -d: -f2` - """ - - run_robot_systest( - container_name, - params.TEST_NAME, - osmHostname, - prometheusHostname, - prometheusPort, - params.ROBOT_VIM, - params.KUBECONFIG, - params.CLOUDS, - hostfile, - jujuPassword) + } else { + hostfile=null } + + jujuPassword=sshCommand remote: remote, command: """ + echo `juju gui 2>&1 | grep password | cut -d: -f2` + """ + + run_robot_systest( + container_name, + params.ROBOT_TAG_NAME, + osmHostname, + prometheusHostname, + prometheusPort, + params.ROBOT_VIM, + params.ROBOT_PORT_MAPPING_VIM, + params.KUBECONFIG, + params.CLOUDS, + hostfile, + jujuPassword, + SSH_KEY, + params.ROBOT_PASS_THRESHOLD, + params.ROBOT_UNSTABLE_THRESHOLD + ) } // stage("System Integration Test") } finally { stage("Archive Container Logs") { // Archive logs to containers_logs.txt archive_logs(remote) - if ( ! currentBuild.result.equals('UNSTABLE') && ! currentBuild.result.equals('FAILURE')) { + if ( ! currentBuild.result.equals('FAILURE') ) { stage_archive = keep_artifacts } else { println ("Systest test failed, throwing error") @@ -540,7 +588,7 @@ EOF""" } } } - } // if ( params.DO_STAGE_4 ) + } // if ( params.DO_ROBOT ) if ( params.SAVE_ARTIFACTS_OVERRIDE || stage_archive ) { stage("Archive") { @@ -562,7 +610,9 @@ EOF""" parallelSteps[module] = { dir("$module") { - sh "docker tag opensourcemano/${moduleName}:${moduleTag} opensourcemano/${moduleName}:${dockerTag}" + sh("docker pull ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}") + sh("""docker tag opensourcemano/${moduleName}:${moduleTag} \ + opensourcemano/${moduleName}:${dockerTag}""") sh "docker push opensourcemano/${moduleName}:${dockerTag}" } } @@ -570,31 +620,32 @@ EOF""" parallel parallelSteps } - stage("Snap promotion") { - def snaps = ["osmclient"] - sh "snapcraft login --with ~/.snapcraft/config" - for (snap in snaps) { - channel="latest/" - if (BRANCH_NAME.startsWith("v")) { - channel=BRANCH_NAME.substring(1)+"/" - } else if (BRANCH_NAME!="master") { - channel+="/"+BRANCH_NAME.replaceAll('/','-') - } - track=channel+"edge\\*" - edge_rev=sh(returnStdout: true, - script: "snapcraft revisions $snap | " + - "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "edge rev is $edge_rev" - track=channel+"beta\\*" - beta_rev=sh(returnStdout: true, - script: "snapcraft revisions $snap | " + - "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "beta rev is $beta_rev" - - if ( edge_rev != beta_rev ) { - print "Promoting $edge_rev to beta in place of $beta_rev" - beta_track=channel+"beta" - sh "snapcraft release $snap $edge_rev $beta_track" + stage('Snap promotion') { + withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { + snaps = ['osmclient'] + for (snap in snaps) { + channel = 'latest/' + if (BRANCH_NAME.startsWith('v')) { + channel = BRANCH_NAME.substring(1) + '/' + } else if (BRANCH_NAME != 'master') { + channel += '/' + BRANCH_NAME.replaceAll('/', '-') + } + track = channel + 'edge\\*' + edge_rev = sh(returnStdout: true, + script: "snapcraft revisions $snap | " + + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + print "edge rev is $edge_rev" + track = channel + 'beta\\*' + beta_rev = sh(returnStdout: true, + script: "snapcraft revisions $snap | " + + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + print "beta rev is $beta_rev" + + if (edge_rev != beta_rev) { + print "Promoting $edge_rev to beta in place of $beta_rev" + beta_track = channel + 'beta' + sh "snapcraft release $snap $edge_rev $beta_track" + } } } } // stage("Snap promotion")