X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=jenkins%2Fci-pipelines%2Fci_stage_3.groovy;h=a65b8b7085690d59a1d852cb1917a567c677857b;hb=8585f3369306457cb2c3918f8bb5e47d5796dd79;hp=83671d14bfd6ad22759ac097aeeeeb75593698d7;hpb=8c76829f25eb1b8ba2c2c8b00a606476ae0e91b0;p=osm%2Fdevops.git diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 83671d14..a65b8b70 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -176,6 +176,7 @@ node("${params.NODE}") { INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000' APT_PROXY = 'http://172.21.1.1:3142' SSH_KEY = '~/hive/cicd_rsa' + ARCHIVE_LOGS_FLAG = false sh 'env' tag_or_branch = params.GERRIT_BRANCH.replaceAll(/\./, '') @@ -316,9 +317,10 @@ node("${params.NODE}") { repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0));' + 'print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim() - repo_base_url = ci_helper.start_http_server(pwd, http_server_name, repo_port) + internal_docker_http_server_url = ci_helper.start_http_server(pwd, http_server_name, repo_port) NODE_IP_ADDRESS = sh(returnStdout: true, script: "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim() + ci_helper.check_status_http_server(NODE_IP_ADDRESS, repo_port) } // Unpack devops package into temporary location so that we use it from upstream if it was part of a patch @@ -418,15 +420,45 @@ node("${params.NODE}") { println("Waiting for VM at ${IP_ADDRESS} to be reachable") alive = false - while (!alive) { - output = sh(returnStdout: true, script: "sleep 1 ; nc -zv ${IP_ADDRESS} 22 2>&1 || true").trim() - println("output is [$output]") - alive = output.contains('succeeded') + timeout(time: 1, unit: 'MINUTES') { + while (!alive) { + output = sh( + returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "-o ConnectTimeout=5 ubuntu@${IP_ADDRESS} 'echo Alive'") + alive = (output == 0) + } } println('VM is ready and accepting ssh connections') } // stage("Spawn Remote VM") /////////////////////////////////////////////////////////////////////////////////////// +// Checks before installation +/////////////////////////////////////////////////////////////////////////////////////// + stage('Checks before installation') { + remote = [ + name: containerName, + host: IP_ADDRESS, + user: 'ubuntu', + identityFile: SSH_KEY, + allowAnyHosts: true, + logLevel: 'INFO', + pty: true + ] + + // Ensure the VM is ready + sshCommand remote: remote, command: 'cloud-init status --wait' + // Force time sync to avoid clock drift and invalid certificates + sshCommand remote: remote, command: 'sudo apt-get update' + sshCommand remote: remote, command: 'sudo apt-get install -y chrony' + sshCommand remote: remote, command: 'sudo service chrony stop' + sshCommand remote: remote, command: 'sudo chronyd -vq' + sshCommand remote: remote, command: 'sudo service chrony start' + + } // stage("Checks before installation") +/////////////////////////////////////////////////////////////////////////////////////// // Installation /////////////////////////////////////////////////////////////////////////////////////// stage('Install') { @@ -453,24 +485,15 @@ node("${params.NODE}") { repo_base_url = "-u http://${NODE_IP_ADDRESS}:${repo_port}" } - remote.with { - name = containerName - host = IP_ADDRESS - user = 'ubuntu' - identityFile = SSH_KEY - allowAnyHosts = true - logLevel = 'INFO' - pty = true - } - - // Force time sync to avoid clock drift and invalid certificates - sshCommand remote: remote, command: ''' - sudo apt update - sudo apt install -y ntp - sudo service ntp stop - sudo ntpd -gq - sudo service ntp start - ''' + remote = [ + name: containerName, + host: IP_ADDRESS, + user: 'ubuntu', + identityFile: SSH_KEY, + allowAnyHosts: true, + logLevel: 'INFO', + pty: true + ] sshCommand remote: remote, command: ''' wget https://osm-download.etsi.org/ftp/osm-11.0-eleven/install_osm.sh @@ -528,6 +551,8 @@ node("${params.NODE}") { // Health check of installed OSM in remote vm /////////////////////////////////////////////////////////////////////////////////////// stage('OSM Health') { + // if this point is reached, logs should be archived + ARCHIVE_LOGS_FLAG = true stackName = 'osm' sshCommand remote: remote, command: """ /usr/share/osm-devops/installers/osm_health.sh -k -s ${stackName} @@ -577,9 +602,7 @@ EOF""" ) } // stage("System Integration Test") } finally { - stage('Archive Container Logs') { - // Archive logs to containers_logs.txt - archive_logs(remote) + stage('After System Integration test') { if (currentBuild.result != 'FAILURE') { stage_archive = keep_artifacts } else { @@ -594,9 +617,6 @@ EOF""" if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) { stage('Archive') { - sh "echo ${containerName} > build_version.txt" - archiveArtifacts artifacts: 'build_version.txt', fingerprint: true - // Archive the tested repo dir("${RELEASE_DIR}") { ci_helper.archive(params.ARTIFACTORY_SERVER, RELEASE, GERRIT_BRANCH, 'tested') @@ -605,10 +625,10 @@ EOF""" stage('Publish to Dockerhub') { parallelSteps = [:] for (buildStep in containerList) { - module = buildStep - moduleName = buildStep.toLowerCase() - dockerTag = params.DOCKER_TAG - moduleTag = containerName + def module = buildStep + def moduleName = buildStep.toLowerCase() + def dockerTag = params.DOCKER_TAG + def moduleTag = containerName parallelSteps[module] = { dir("$module") { @@ -620,68 +640,143 @@ EOF""" } parallel parallelSteps } - stage('Snap promotion') { - snaps = ['osmclient'] - sh 'snapcraft login --with ~/.snapcraft/config' - for (snap in snaps) { - channel = 'latest/' + withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { + snaps = ['osmclient'] + for (snap in snaps) { + channel = 'latest/' + if (BRANCH_NAME.startsWith('v')) { + channel = BRANCH_NAME.substring(1) + '/' + } else if (BRANCH_NAME != 'master') { + channel += '/' + BRANCH_NAME.replaceAll('/', '-') + } + track = channel + 'edge\\*' + edge_rev = sh(returnStdout: true, + script: "snapcraft revisions $snap | " + + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + track = channel + 'beta\\*' + beta_rev = sh(returnStdout: true, + script: "snapcraft revisions $snap | " + + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + + print "Edge: $edge_rev, Beta: $beta_rev" + + if (edge_rev != beta_rev) { + print "Promoting $edge_rev to beta in place of $beta_rev" + beta_track = channel + 'beta' + sh "snapcraft release $snap $edge_rev $beta_track" + } + } + } + } // stage('Snap promotion') + stage('Charm promotion') { + charms = [ + 'osm', // bundle + 'osm-ha', // bundle + 'osm-grafana', + 'osm-mariadb', + 'mongodb-exporter-k8s', + 'mysqld-exporter-k8s', + 'osm-lcm', + 'osm-mon', + 'osm-nbi', + 'osm-ng-ui', + 'osm-pol', + 'osm-ro', + 'osm-prometheus', + 'osm-vca-integrator', + ] + for (charm in charms) { + + channel = 'latest' if (BRANCH_NAME.startsWith('v')) { - channel = BRANCH_NAME.substring(1) + '/' + channel = BRANCH_NAME.substring(1) } else if (BRANCH_NAME != 'master') { channel += '/' + BRANCH_NAME.replaceAll('/', '-') } - track = channel + 'edge\\*' - edge_rev = sh(returnStdout: true, - script: "snapcraft revisions $snap | " + - "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "edge rev is $edge_rev" - track = channel + 'beta\\*' - beta_rev = sh(returnStdout: true, - script: "snapcraft revisions $snap | " + - "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "beta rev is $beta_rev" - - if (edge_rev != beta_rev) { - print "Promoting $edge_rev to beta in place of $beta_rev" - beta_track = channel + 'beta' - sh "snapcraft release $snap $edge_rev $beta_track" + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + sh "charmcraft status $charm --format json > ${charm}.json" + isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int + if (isCharm) { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/beta\")| .version'|head -1" + } else { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/beta\")| .version'|head -1" + } + // edge/merged is used in place of /edge as 10.1.0 LTS uses latest/edge + edge_rev = sh(returnStdout: true, script: jqScriptEdge).trim() + beta_rev = sh(returnStdout: true, script: jqScriptBeta).trim() + try { edge_rev = edge_rev as int } catch (NumberFormatException nfe) {edge_rev = 0} + try { beta_rev = beta_rev as int } catch (NumberFormatException nfe) {beta_rev = 0} + + print "Edge: $edge_rev, Beta: $beta_rev" + + if (edge_rev > beta_rev) { + print "Promoting $edge_rev to beta in place of $beta_rev" + beta_track = channel + 'beta' + sh "charmcraft release ${charm} --revision=${edge_rev} --channel=${channel}/beta" + } + } } - } // stage('Snap promotion') + } // stage('Charm promotion') } // if (params.DO_DOCKERPUSH) } // stage('Archive') } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) } // dir(OSM_DEVOPS) } finally { - if ( params.DO_INSTALL && server_id != null) { - delete_vm = true - if (error && params.SAVE_CONTAINER_ON_FAIL ) { - delete_vm = false - } - if (!error && params.SAVE_CONTAINER_ON_PASS ) { - delete_vm = false - } + stage('Archive Container Logs') { + if ( ARCHIVE_LOGS_FLAG ) { + try { + // Archive logs + remote = [ + name: containerName, + host: IP_ADDRESS, + user: 'ubuntu', + identityFile: SSH_KEY, + allowAnyHosts: true, + logLevel: 'INFO', + pty: true + ] + println('Archiving container logs') + archive_logs(remote) + } catch (Exception e) { + println('Error fetching logs: '+ e.getMessage()) + } + } // end if ( ARCHIVE_LOGS_FLAG ) + } + stage('Cleanup') { + if ( params.DO_INSTALL && server_id != null) { + delete_vm = true + if (error && params.SAVE_CONTAINER_ON_FAIL ) { + delete_vm = false + } + if (!error && params.SAVE_CONTAINER_ON_PASS ) { + delete_vm = false + } - if ( delete_vm ) { - if (server_id != null) { - println("Deleting VM: $server_id") - sh """#!/bin/sh -e - for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done - openstack server delete ${server_id} - """ - } else { - println("Saved VM $server_id in ETSI VIM") + if ( delete_vm ) { + if (server_id != null) { + println("Deleting VM: $server_id") + sh """#!/bin/sh -e + for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done + openstack server delete ${server_id} + """ + } else { + println("Saved VM $server_id in ETSI VIM") + } } } - } - if ( http_server_name != null ) { - sh "docker stop ${http_server_name} || true" - sh "docker rm ${http_server_name} || true" - } + if ( http_server_name != null ) { + sh "docker stop ${http_server_name} || true" + sh "docker rm ${http_server_name} || true" + } - if ( devopstempdir != null ) { - sh "rm -rf ${devopstempdir}" + if ( devopstempdir != null ) { + sh "rm -rf ${devopstempdir}" + } } } }