jenkins/ci-pipelines/ci_stage_3.groovy

   1 /* Copyright 2017 Sandvine
   2  *
   3  * All Rights Reserved.
   4  *
   5  *   Licensed under the Apache License, Version 2.0 (the "License"); you may
   6  *   not use this file except in compliance with the License. You may obtain
   7  *   a copy of the License at
   8  *
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *   Unless required by applicable law or agreed to in writing, software
  12  *   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  *   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  *   License for the specific language governing permissions and limitations
  15  *   under the License.
  16  */
  17
  18 properties([
  19     parameters([
  20         string(defaultValue: env.GERRIT_BRANCH, description: '', name: 'GERRIT_BRANCH'),
  21         string(defaultValue: 'system', description: '', name: 'NODE'),
  22         string(defaultValue: '', description: '', name: 'BUILD_FROM_SOURCE'),
  23         string(defaultValue: 'unstable', description: '', name: 'REPO_DISTRO'),
  24         string(defaultValue: '', description: '', name: 'COMMIT_ID'),
  25         string(defaultValue: '-stage_2', description: '', name: 'UPSTREAM_SUFFIX'),
  26         string(defaultValue: 'pubkey.asc', description: '', name: 'REPO_KEY_NAME'),
  27         string(defaultValue: 'release', description: '', name: 'RELEASE'),
  28         string(defaultValue: '', description: '', name: 'UPSTREAM_JOB_NAME'),
  29         string(defaultValue: '', description: '', name: 'UPSTREAM_JOB_NUMBER'),
  30         string(defaultValue: '', description: '', name: 'UPSTREAM_JOB_NUMBER'),
  31         string(defaultValue: 'OSMETSI', description: '', name: 'GPG_KEY_NAME'),
  32         string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'),
  33         string(defaultValue: 'osm-stage_4', description: '', name: 'DOWNSTREAM_STAGE_NAME'),
  34         string(defaultValue: 'testing-daily', description: '', name: 'DOCKER_TAG'),
  35         booleanParam(defaultValue: true, description: '', name: 'SAVE_CONTAINER_ON_FAIL'),
  36         booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'),
  37         booleanParam(defaultValue: true, description: '', name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS'),
  38         booleanParam(defaultValue: true, description: '', name: 'DO_STAGE_4'),
  39         booleanParam(defaultValue: true, description: '',  name: 'DO_BUILD'),
  40         booleanParam(defaultValue: true, description: '', name: 'DO_INSTALL'),
  41         booleanParam(defaultValue: true, description: '', name: 'DO_SMOKE'),
  42         booleanParam(defaultValue: true, description: '', name: 'DO_DOCKERPUSH'),
  43         booleanParam(defaultValue: false, description: '', name: 'SAVE_ARTIFACTS_OVERRIDE'),
  44         string(defaultValue: '/home/jenkins/hive/openstack-etsi.rc', description: '', name: 'HIVE_VIM_1'),
  45         booleanParam(defaultValue: false, description: '', name: 'DO_ROBOT'),
  46         string(defaultValue: 'sanity', description: 'sanity/regression are the options', name: 'TEST_NAME'),
  47         string(defaultValue: '/home/jenkins/hive/robot-systest.cfg', description: '', name: 'ROBOT_VIM'),
  48         string(defaultValue: '/home/jenkins/hive/kubeconfig.yaml', description: '', name: 'KUBECONFIG'),
  49         string(defaultValue: '/home/jenkins/hive/clouds.yaml', description: '', name: 'CLOUDS'),
  50         string(defaultValue: 'Default', description: '', name: 'INSTALLER'),
  51     ])
  52 ])
  53
  54 def uninstall_osm(stackName) {
  55     sh """
  56          export OSM_USE_LOCAL_DEVOPS=true
  57          export PATH=$PATH:/snap/bin
  58          installers/full_install_osm.sh -y -c swarm -w /tmp/osm -t ${stackName} -s ${stackName} --test --nolxd --nodocker --nojuju --nohostports --nohostclient --uninstall
  59        """
  60 }
  61
  62 def run_systest(stackName,tagName,testName,envfile=null) {
  63     tempdir = sh(returnStdout: true, script: "mktemp -d").trim()
  64     if ( !envfile )
  65     {
  66         sh(script: "touch ${tempdir}/env")
  67         envfile="${tempdir}/env"
  68     }
  69     sh "docker run --network net${stackName} --env-file ${envfile} -v ${tempdir}:/usr/share/osm-devops/systest/reports opensourcemano/osmclient:${tagName} make -C /usr/share/osm-devops/systest ${testName}"
  70     sh "cp ${tempdir}/* ."
  71     junit  '*.xml'
  72 }
  73
  74 def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus_port=null,envfile=null,kubeconfig=null,clouds=null,hostfile=null,jujuPassword=null) {
  75     tempdir = sh(returnStdout: true, script: "mktemp -d").trim()
  76     if ( !envfile )
  77     {
  78         sh(script: "touch ${tempdir}/env")
  79         envfile="${tempdir}/env"
  80     }
  81     PROMETHEUS_PORT_VAR = ""
  82     if ( prometheusPort != null) {
  83         PROMETHEUS_PORT_VAR = "--env PROMETHEUS_PORT="+prometheusPort
  84     }
  85     hostfilemount=""
  86     if ( hostfile ) {
  87         hostfilemount="-v "+hostfile+":/etc/hosts"
  88     }
  89
  90     JUJU_PASSWORD_VAR = ""
  91     if ( jujuPassword != null) {
  92         JUJU_PASSWORD_VAR = "--env JUJU_PASSWORD="+jujuPassword
  93     }
  94
  95     try {
  96         sh "docker run --env OSM_HOSTNAME=${osmHostname} --env PROMETHEUS_HOSTNAME=${prometheusHostname} ${PROMETHEUS_PORT_VAR} ${JUJU_PASSWORD_VAR} --env-file ${envfile} -v ${clouds}:/etc/openstack/clouds.yaml -v ${kubeconfig}:/root/.kube/config -v ${tempdir}:/robot-systest/reports ${hostfilemount} opensourcemano/tests:${tagName} -c -t ${testName}"
  97     } finally {
  98         sh "cp ${tempdir}/* ."
  99         outputDirectory = sh(returnStdout: true, script: "pwd").trim()
 100         println ("Present Directory is : ${outputDirectory}")
 101         step([
 102             $class : 'RobotPublisher',
 103             outputPath : "${outputDirectory}",
 104             outputFileName : "*.xml",
 105             disableArchiveOutput : false,
 106             reportFileName : "report.html",
 107             logFileName : "log.html",
 108             passThreshold : 0,
 109             unstableThreshold: 0,
 110             otherFiles : "*.png",
 111         ])
 112     }
 113 }
 114
 115 def archive_logs(remote) {
 116
 117 -def archive_logs(stackName) {
 118 -    sh "docker service ls |grep \"${stackName}\"| awk '{print \$2}'| xargs -iy docker service logs y --timestamps > containers_logs.txt 2>&1"
 119 -    archiveArtifacts artifacts: 'containers_logs.txt'
 120     sshCommand remote: remote, command: '''mkdir -p logs'''
 121     if (useCharmedInstaller) {
 122         sshCommand remote: remote, command: '''
 123             for container in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do
 124                 logfile=`echo $container | cut -d- -f1`
 125                 echo "Extracting log for $logfile"
 126                 kubectl logs -n osm $container --timestamps=true 2>&1 > logs/$logfile.log
 127             done
 128         '''
 129     } else {
 130         sshCommand remote: remote, command: '''
 131             for service in `docker service ls| grep \"${stackName}\"| awk '{print \$2}`; do
 132                 echo "Extracting log for $service"
 133                 docker service logs $service --timestamps 2>&1 > logs/$service.log
 134             done
 135         '''
 136     }
 137
 138     sh "rm -rf logs"
 139     sshCommand remote: remote, command: '''ls -al logs'''
 140     sshGet remote: remote, from: 'logs', into: '.', override: true
 141     sh "cp logs/* ."
 142     archiveArtifacts artifacts: '*.log'
 143 }
 144
 145 def get_value(key, output) {
 146     for (String line : output.split( '\n' )) {
 147         data = line.split( '\\|' )
 148         if (data.length > 1) {
 149             if ( data[1].trim() == key ) {
 150                 return data[2].trim()
 151             }
 152         }
 153     }
 154 }
 155
 156 node("${params.NODE}") {
 157
 158     INTERNAL_DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/'
 159     INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000'
 160     SSH_KEY = '~/hive/cicd_rsa'
 161     sh 'env'
 162
 163     tag_or_branch = params.GERRIT_BRANCH.replaceAll(/\./,"")
 164
 165     stage("Checkout") {
 166         checkout scm
 167     }
 168
 169     ci_helper = load "jenkins/ci-pipelines/ci_helper.groovy"
 170
 171     def upstream_main_job = params.UPSTREAM_SUFFIX
 172
 173     // upstream jobs always use merged artifacts
 174     upstream_main_job += '-merge'
 175     container_name_prefix = "osm-${tag_or_branch}"
 176     container_name = "${container_name_prefix}"
 177
 178     keep_artifacts = false
 179     if ( JOB_NAME.contains('merge') ) {
 180         container_name += "-merge"
 181
 182         // On a merge job, we keep artifacts on smoke success
 183         keep_artifacts = params.SAVE_ARTIFACTS_ON_SMOKE_SUCCESS
 184     }
 185     container_name += "-${BUILD_NUMBER}"
 186
 187     // Copy the artifacts from the upstream jobs
 188     stage("Copy Artifacts") {
 189         // cleanup any previous repo
 190         sh 'rm -rf repo'
 191         dir("repo") {
 192             // grab all stable upstream builds based on the
 193
 194             dir("${RELEASE}") {
 195                 def list = ["RO", "openvim", "osmclient", "IM", "devops", "MON", "N2VC", "NBI", "common", "LCM", "POL", "LW-UI", "NG-UI", "PLA", "tests"]
 196                 for (component in list) {
 197                     step ([$class: 'CopyArtifact',
 198                            projectName: "${component}${upstream_main_job}/${GERRIT_BRANCH}"])
 199
 200                     // grab the build name/number
 201                     build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER')
 202
 203                     // grab the archives from the stage_2 builds (ie. this will be the artifacts stored based on a merge)
 204                     ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${component}${upstream_main_job} :: ${GERRIT_BRANCH}", build_num)
 205
 206                     // cleanup any prevously defined dists
 207                     sh "rm -rf dists"
 208                 }
 209
 210                 // check if an upstream artifact based on specific build number has been requested
 211                 // This is the case of a merge build and the upstream merge build is not yet complete (it is not deemed
 212                 // a successful build yet). The upstream job is calling this downstream job (with the its build artifiact)
 213                 if ( params.UPSTREAM_JOB_NAME ) {
 214                     step ([$class: 'CopyArtifact',
 215                            projectName: "${params.UPSTREAM_JOB_NAME}",
 216                            selector: [$class: 'SpecificBuildSelector', buildNumber: "${params.UPSTREAM_JOB_NUMBER}"]
 217                           ])
 218
 219                     build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER')
 220                     component = ci_helper.get_mdg_from_project(ci_helper.get_env_value('build.env','GERRIT_PROJECT'))
 221
 222                     // the upstream job name contains suffix with the project. Need this stripped off
 223                     def project_without_branch = params.UPSTREAM_JOB_NAME.split('/')[0]
 224
 225                     // Remove the previous artifact for this component. Use the new upstream artifact
 226                     sh "rm -rf pool/${component}"
 227
 228                     ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${project_without_branch} :: ${GERRIT_BRANCH}", build_num)
 229
 230                     sh "rm -rf dists"
 231                 }
 232
 233                 // sign all the components
 234                 for (component in list) {
 235                     sh "dpkg-sig --sign builder -k ${GPG_KEY_NAME} pool/${component}/*"
 236                 }
 237
 238                 // now create the distro
 239                 for (component in list) {
 240                     sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/"
 241                     sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages"
 242                     sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages"
 243                 }
 244
 245                 // create and sign the release file
 246                 sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release"
 247                 sh "gpg --yes -abs -u ${GPG_KEY_NAME} -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release"
 248
 249                 // copy the public key into the release folder
 250                 // this pulls the key from the home dir of the current user (jenkins)
 251                 sh "cp ~/${REPO_KEY_NAME} 'OSM ETSI Release Key.gpg'"
 252                 sh "cp ~/${REPO_KEY_NAME} ."
 253
 254                 // merge the change logs
 255                 sh """
 256                    rm -f changelog/changelog-osm.html
 257                    [ ! -d changelog ] || for mdgchange in \$(ls changelog); do cat changelog/\$mdgchange >> changelog/changelog-osm.html; done
 258                    """
 259                 RELEASE_DIR = sh(returnStdout:true,  script: 'pwd').trim()
 260             }
 261             // start an apache server to serve up the images
 262             http_server_name = "${container_name}-apache"
 263
 264             pwd = sh(returnStdout:true,  script: 'pwd').trim()
 265             repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim()
 266             repo_base_url = ci_helper.start_http_server(pwd,http_server_name,repo_port)
 267             NODE_IP_ADDRESS=sh(returnStdout: true, script:
 268                 "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim()
 269         }
 270
 271         // now pull the devops package and install in temporary location
 272         tempdir = sh(returnStdout: true, script: "mktemp -d").trim()
 273         osm_devops_dpkg = sh(returnStdout: true, script: "find . -name osm-devops*.deb").trim()
 274         sh "dpkg -x ${osm_devops_dpkg} ${tempdir}"
 275         OSM_DEVOPS="${tempdir}/usr/share/osm-devops"
 276         println("Repo base URL=${repo_base_url}")
 277     }
 278
 279     dir(OSM_DEVOPS) {
 280         def remote = [:]
 281         error = null
 282
 283         if ( params.DO_BUILD ) {
 284             stage("Build") {
 285                 sh "make -C docker clean"
 286                 sh "make -C docker -j `nproc` Q= CMD_DOCKER_ARGS= TAG=${container_name} RELEASE=${params.RELEASE} REPOSITORY_BASE=${repo_base_url} REPOSITORY_KEY=${params.REPO_KEY_NAME} REPOSITORY=${params.REPO_DISTRO}"
 287             }
 288
 289             stage("Push to internal registry") {
 290                 withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry',
 291                                 usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
 292                     sh "docker login ${INTERNAL_DOCKER_REGISTRY} -u ${USERNAME} -p ${PASSWORD}"
 293                 }
 294                 sh "make -C docker push INPUT_TAG=${container_name} TAG=${container_name} DOCKER_REGISTRY=${INTERNAL_DOCKER_REGISTRY}"
 295             }
 296
 297         }
 298
 299         try {
 300             useCharmedInstaller = params.INSTALLER.equalsIgnoreCase("charmed")
 301
 302             if ( params.DO_INSTALL ) {
 303
 304                 stage("Spawn Remote VM") {
 305                     println("Launching new VM")
 306                     output=sh(returnStdout: true, script: """#!/bin/sh -e
 307                         for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done
 308                         openstack server create --flavor m1.xlarge \
 309                                                 --image ubuntu18.04 \
 310                                                 --key-name CICD \
 311                                                 --nic net-id=osm-ext \
 312                                                 ${container_name}
 313                     """).trim()
 314
 315                     server_id = get_value('id', output)
 316
 317                     if (server_id == null) {
 318                         println("VM launch output: ")
 319                         println(output)
 320                         throw new Exception("VM Launch failed")
 321                     }
 322                     println("Target VM is ${server_id}, waiting for IP address to be assigned")
 323
 324                     IP_ADDRESS = ""
 325
 326                     while (IP_ADDRESS == "") {
 327                         output=sh(returnStdout: true, script: """#!/bin/sh -e
 328                             for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done
 329                             openstack server show ${server_id}
 330                         """).trim()
 331                         IP_ADDRESS = get_value('addresses', output)
 332                     }
 333                     IP_ADDRESS = IP_ADDRESS.split('=')[1]
 334                     println("Waiting for VM at ${IP_ADDRESS} to be reachable")
 335
 336                     alive = false
 337                     while (! alive) {
 338                         output=sh(returnStdout: true, script: "sleep 1 ; nc -zv ${IP_ADDRESS} 22 2>&1 || true").trim()
 339                         println("output is [$output]")
 340                         alive = output.contains("succeeded")
 341                     }
 342                     println("VM is ready and accepting ssh connections")
 343                 }
 344
 345                 stage("Install") {
 346                     commit_id = ''
 347                     repo_distro = ''
 348                     repo_key_name = ''
 349                     release = ''
 350
 351                     if ( params.COMMIT_ID )
 352                     {
 353                         commit_id = "-b ${params.COMMIT_ID}"
 354                     }
 355
 356                     if ( params.REPO_DISTRO )
 357                     {
 358                         repo_distro = "-r ${params.REPO_DISTRO}"
 359                     }
 360
 361                     if ( params.REPO_KEY_NAME )
 362                     {
 363                         repo_key_name = "-k ${params.REPO_KEY_NAME}"
 364                     }
 365
 366                     if ( params.RELEASE )
 367                     {
 368                         release = "-R ${params.RELEASE}"
 369                     }
 370
 371                     if ( params.REPOSITORY_BASE )
 372                     {
 373                         repo_base_url = "-u ${params.REPOSITORY_BASE}"
 374                     }
 375                     else
 376                     {
 377                         repo_base_url = "-u http://${NODE_IP_ADDRESS}:${repo_port}"
 378                     }
 379
 380                     remote.name = container_name
 381                     remote.host = IP_ADDRESS
 382                     remote.user = 'ubuntu'
 383                     remote.identityFile = SSH_KEY
 384                     remote.allowAnyHosts = true
 385                     remote.logLevel = 'INFO'
 386                     remote.pty = true
 387
 388                     sshCommand remote: remote, command: """
 389                         wget https://osm-download.etsi.org/ftp/osm-8.0-eight/install_osm.sh
 390                         chmod +x ./install_osm.sh
 391                         sed -i '1 i\\export PATH=/snap/bin:\${PATH}' ~/.bashrc
 392                     """
 393
 394                     if ( useCharmedInstaller ) {
 395
 396                         // Use local proxy for docker hub
 397                         sshCommand remote: remote, command: '''
 398                             sudo snap install microk8s --classic --channel=1.19/stable
 399                             sudo sed -i "s|https://registry-1.docker.io|http://172.21.1.1:5000|" /var/snap/microk8s/current/args/containerd-template.toml
 400                             sudo systemctl restart snap.microk8s.daemon-containerd.service
 401                             sudo snap alias microk8s.kubectl kubectl
 402                         '''
 403
 404                         withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry',
 405                                         usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
 406                             sshCommand remote: remote, command: """
 407                                 ./install_osm.sh -y \
 408                                     ${repo_base_url} \
 409                                     ${repo_key_name} \
 410                                     ${release} -r unstable \
 411                                     --charmed  \
 412                                     --registry ${USERNAME}:${PASSWORD}@${INTERNAL_DOCKER_REGISTRY} \
 413                                     --tag ${container_name}
 414                             """
 415                         }
 416                         prometheusHostname = "prometheus."+IP_ADDRESS+".xip.io"
 417                         prometheusPort = 80
 418                         osmHostname = "nbi."+IP_ADDRESS+".xip.io:443"
 419                     } else {
 420                         // Run -k8s installer here specifying internal docker registry and docker proxy
 421                         withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry',
 422                                         usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
 423                             sshCommand remote: remote, command: """
 424                                 ./install_osm.sh -y \
 425                                     ${repo_base_url} \
 426                                     ${repo_key_name} \
 427                                     ${release} -r unstable \
 428                                     -d ${USERNAME}:${PASSWORD}@${INTERNAL_DOCKER_REGISTRY} \
 429                                     -p ${INTERNAL_DOCKER_PROXY} \
 430                                     -t ${container_name}
 431                             """
 432                         }
 433                         prometheusHostname = IP_ADDRESS
 434                         prometheusPort = 9091
 435                         osmHostname = IP_ADDRESS
 436                     }
 437                 }
 438             }
 439
 440             stage_archive = false
 441             if ( params.DO_SMOKE ) {
 442                 stage("OSM Health") {
 443                     stackName = "osm"
 444                     sshCommand remote: remote, command: """
 445                         /usr/share/osm-devops/installers/osm_health.sh -k -s ${stackName}
 446                     """
 447                 }
 448             }
 449
 450             if ( params.DO_STAGE_4 ) {
 451                 // override stage_archive to only archive on stable
 452                 stage_archive = false
 453                 try {
 454                     stage("System Integration Test") {
 455                         if ( params.DO_ROBOT ) {
 456                             if( useCharmedInstaller ) {
 457                                 tempdir = sh(returnStdout: true, script: "mktemp -d").trim()
 458                                 sh(script: "touch ${tempdir}/hosts")
 459                                 hostfile="${tempdir}/hosts"
 460                                 sh """cat << EOF > ${hostfile}
 461 127.0.0.1           localhost
 462 ${remote.host}      prometheus.${remote.host}.xip.io nbi.${remote.host}.xip.io
 463 EOF"""
 464                             } else {
 465                                 hostfile=null
 466                             }
 467
 468                             jujuPassword=sshCommand remote: remote, command: """
 469                                 echo `juju gui 2>&1 | grep password | cut -d: -f2`
 470                             """
 471
 472                             run_robot_systest(
 473                                 container_name,
 474                                 params.TEST_NAME,
 475                                 osmHostname,
 476                                 prometheusHostname,
 477                                 prometheusPort,
 478                                 params.ROBOT_VIM,
 479                                 params.KUBECONFIG,
 480                                 params.CLOUDS,
 481                                 hostfile,
 482                                 jujuPassword)
 483                         }
 484                     }
 485                 } finally {
 486                     stage("Archive Container Logs") {
 487                         // Archive logs to containers_logs.txt
 488                         archive_logs(remote)
 489                         if ( ! currentBuild.result.equals('UNSTABLE') && ! currentBuild.result.equals('FAILURE')) {
 490                             stage_archive = keep_artifacts
 491                         } else {
 492                             println ("Systest test failed, throwing error")
 493                             error = new Exception("Systest test failed")
 494                             currentBuild.result = 'FAILURE'
 495                             throw error
 496                         }
 497                     }
 498                 }
 499             }
 500
 501             // override to save the artifacts
 502             if ( params.SAVE_ARTIFACTS_OVERRIDE || stage_archive ) {
 503                 stage("Archive") {
 504                     sh "echo ${container_name} > build_version.txt"
 505                     archiveArtifacts artifacts: "build_version.txt", fingerprint: true
 506
 507                     // Archive the tested repo
 508                     dir("${RELEASE_DIR}") {
 509                         ci_helper.archive(params.ARTIFACTORY_SERVER,RELEASE,GERRIT_BRANCH,'tested')
 510                     }
 511                     if ( params.DO_DOCKERPUSH ) {
 512                         stage("Docker Push") {
 513                             sh "make -C docker push INPUT_TAG=${container_name} TAG=${params.DOCKER_TAG}"
 514                         }
 515
 516                         stage("Snap promotion") {
 517                             def snaps = ["osmclient"]
 518                             for (snap in snaps) {
 519                                 channel=""
 520                                 if (BRANCH_NAME.startsWith("v")) {
 521                                     channel=BRANCH_NAME.substring(1)+"/"
 522                                 } else if (BRANCH_NAME!="master") {
 523                                     channel+="/"+BRANCH_NAME.replaceAll('/','-')
 524                                 }
 525                                 track=channel+"edge\\*"
 526                                 edge_rev=sh(returnStdout: true,
 527                                     script: "sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " +
 528                                     "-w /build snapcore/snapcraft:stable /bin/bash -c " +
 529                                     "\"snapcraft login --with /snapcraft/config &>/dev/null && " +
 530                                     "snapcraft revisions $snap\" | " +
 531                                     "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
 532                                 track=channel+"beta\\*"
 533                                 beta_rev=sh(returnStdout: true,
 534                                     script: "sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " +
 535                                     "-w /build snapcore/snapcraft:stable /bin/bash -c " +
 536                                     "\"snapcraft login --with /snapcraft/config &>/dev/null && " +
 537                                     "snapcraft revisions $snap\" | " +
 538                                     "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
 539
 540                                 if ( edge_rev != beta_rev ) {
 541                                     print "Promoting $edge_rev to beta in place of $beta_rev"
 542                                     beta_track=channel+"beta"
 543                                     sh("sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " +
 544                                         "-w /build snapcore/snapcraft:stable /bin/bash -c " +
 545                                         "\"snapcraft login --with /snapcraft/config &>/dev/null && " +
 546                                         "snapcraft release $snap $edge_rev $beta_track\"")
 547                                 }
 548                             }
 549                         }
 550                     }
 551                 }
 552             }
 553         }
 554         catch(Exception ex) {
 555             error = ex
 556             currentBuild.result = 'FAILURE'
 557             println("Caught error: "+ex)
 558         }
 559         finally {
 560             println("Entered finally block")
 561             if ( params.DO_INSTALL && server_id != null) {
 562                 delete_vm = true
 563                 if (error && params.SAVE_CONTAINER_ON_FAIL ) {
 564                     delete_vm = false
 565                 }
 566                 if (!error && params.SAVE_CONTAINER_ON_PASS ) {
 567                     delete_vm = false
 568                 }
 569
 570                 if ( delete_vm ) {
 571                     if (server_id != null) {
 572                         println("Deleting VM: $server_id")
 573                         sh """#!/bin/sh -e
 574                             for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done
 575                             openstack server delete ${server_id}
 576                         """
 577                     } else {
 578                         println("Saved VM $server_id in ETSI VIM")
 579                     }
 580                 }
 581             }
 582             sh "docker stop ${http_server_name} || true"
 583             sh "docker rm ${http_server_name} || true"
 584         }
 585     }
 586 }