Bug 2129: Fix logic to always delete VM
[osm/devops.git] / jenkins / ci-pipelines / ci_stage_3.groovy
index c203313..d9d1fbd 100644 (file)
@@ -153,7 +153,8 @@ void archive_logs(Map remote) {
     sshCommand remote: remote, command: '''ls -al logs'''
     sshGet remote: remote, from: 'logs', into: '.', override: true
     sh 'cp logs/* .'
-    archiveArtifacts artifacts: '*.log'
+    sshGet remote: remote, from: 'ens3.pcap', into: 'ens3.pcap', override: true
+    archiveArtifacts artifacts: '*.log, *.pcap'
 }
 
 String get_value(String key, String output) {
@@ -176,6 +177,7 @@ node("${params.NODE}") {
     INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000'
     APT_PROXY = 'http://172.21.1.1:3142'
     SSH_KEY = '~/hive/cicd_rsa'
+    ARCHIVE_LOGS_FLAG = false
     sh 'env'
 
     tag_or_branch = params.GERRIT_BRANCH.replaceAll(/\./, '')
@@ -316,9 +318,10 @@ node("${params.NODE}") {
                 repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0));' +
                                'print(s.getsockname()[1]); s.close()\');',
                                returnStdout: true).trim()
-                repo_base_url = ci_helper.start_http_server(pwd, http_server_name, repo_port)
+                internal_docker_http_server_url = ci_helper.start_http_server(pwd, http_server_name, repo_port)
                 NODE_IP_ADDRESS = sh(returnStdout: true, script:
                     "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim()
+                ci_helper.check_status_http_server(NODE_IP_ADDRESS, repo_port)
             }
 
             // Unpack devops package into temporary location so that we use it from upstream if it was part of a patch
@@ -433,6 +436,29 @@ node("${params.NODE}") {
                 } // stage("Spawn Remote VM")
 
 ///////////////////////////////////////////////////////////////////////////////////////
+// Checks before installation
+///////////////////////////////////////////////////////////////////////////////////////
+                stage('Checks before installation') {
+                    remote = [
+                        name: containerName,
+                        host: IP_ADDRESS,
+                        user: 'ubuntu',
+                        identityFile: SSH_KEY,
+                        allowAnyHosts: true,
+                        logLevel: 'INFO',
+                        pty: true
+                    ]
+
+                    // Force time sync to avoid clock drift and invalid certificates
+                    sshCommand remote: remote, command: 'nohup sudo tcpdump -i ens3 -w ens3.pcap -s 400 & sleep 5'
+                    sshCommand remote: remote, command: 'sudo apt-get update'
+                    sshCommand remote: remote, command: 'sudo apt-get install -y chrony'
+                    sshCommand remote: remote, command: 'sudo service chrony stop'
+                    sshCommand remote: remote, command: 'sudo chronyd -vq'
+                    sshCommand remote: remote, command: 'sudo service chrony start'
+
+                 } // stage("Checks before installation")
+///////////////////////////////////////////////////////////////////////////////////////
 // Installation
 ///////////////////////////////////////////////////////////////////////////////////////
                 stage('Install') {
@@ -469,15 +495,6 @@ node("${params.NODE}") {
                         pty: true
                     ]
 
-                    // Force time sync to avoid clock drift and invalid certificates
-                    sshCommand remote: remote, command: '''
-                        sudo apt update
-                        sudo apt install -y ntp
-                        sudo service ntp stop
-                        sudo ntpd -gq
-                        sudo service ntp start
-                    '''
-
                     sshCommand remote: remote, command: '''
                         wget https://osm-download.etsi.org/ftp/osm-11.0-eleven/install_osm.sh
                         chmod +x ./install_osm.sh
@@ -529,11 +546,17 @@ node("${params.NODE}") {
                         prometheusPort = 9091
                         osmHostname = IP_ADDRESS
                     }
+                    sshCommand remote: remote, command: """
+                        sudo killall tcpdump
+                    """
+
                 } // stage("Install")
 ///////////////////////////////////////////////////////////////////////////////////////
 // Health check of installed OSM in remote vm
 ///////////////////////////////////////////////////////////////////////////////////////
                 stage('OSM Health') {
+                    // if this point is reached, logs should be archived
+                    ARCHIVE_LOGS_FLAG = true
                     stackName = 'osm'
                     sshCommand remote: remote, command: """
                         /usr/share/osm-devops/installers/osm_health.sh -k -s ${stackName}
@@ -549,6 +572,7 @@ node("${params.NODE}") {
             if ( params.DO_ROBOT ) {
                 try {
                     stage('System Integration Test') {
+
                         if (useCharmedInstaller) {
                             tempdir = sh(returnStdout: true, script: 'mktemp -d').trim()
                             sh(script: "touch ${tempdir}/hosts")
@@ -583,9 +607,7 @@ EOF"""
                         )
                     } // stage("System Integration Test")
                 } finally {
-                    stage('Archive Container Logs') {
-                        // Archive logs to containers_logs.txt
-                        archive_logs(remote)
+                    stage('After System Integration test') {
                         if (currentBuild.result != 'FAILURE') {
                             stage_archive = keep_artifacts
                         } else {
@@ -600,9 +622,6 @@ EOF"""
 
             if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) {
                 stage('Archive') {
-                    sh "echo ${containerName} > build_version.txt"
-                    archiveArtifacts artifacts: 'build_version.txt', fingerprint: true
-
                     // Archive the tested repo
                     dir("${RELEASE_DIR}") {
                         ci_helper.archive(params.ARTIFACTORY_SERVER, RELEASE, GERRIT_BRANCH, 'tested')
@@ -628,30 +647,31 @@ EOF"""
                         }
 
                         stage('Snap promotion') {
-                            snaps = ['osmclient']
-                            sh 'snapcraft login --with ~/.snapcraft/config'
-                            for (snap in snaps) {
-                                channel = 'latest/'
-                                if (BRANCH_NAME.startsWith('v')) {
-                                    channel = BRANCH_NAME.substring(1) + '/'
-                                } else if (BRANCH_NAME != 'master') {
-                                    channel += '/' + BRANCH_NAME.replaceAll('/', '-')
-                                }
-                                track = channel + 'edge\\*'
-                                edge_rev = sh(returnStdout: true,
-                                    script: "snapcraft revisions $snap | " +
-                                    "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                print "edge rev is $edge_rev"
-                                track = channel + 'beta\\*'
-                                beta_rev = sh(returnStdout: true,
-                                    script: "snapcraft revisions $snap | " +
-                                    "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                print "beta rev is $beta_rev"
-
-                                if (edge_rev != beta_rev) {
-                                    print "Promoting $edge_rev to beta in place of $beta_rev"
-                                    beta_track = channel + 'beta'
-                                    sh "snapcraft release $snap $edge_rev $beta_track"
+                            withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
+                                snaps = ['osmclient']
+                                for (snap in snaps) {
+                                    channel = 'latest/'
+                                    if (BRANCH_NAME.startsWith('v')) {
+                                        channel = BRANCH_NAME.substring(1) + '/'
+                                    } else if (BRANCH_NAME != 'master') {
+                                        channel += '/' + BRANCH_NAME.replaceAll('/', '-')
+                                    }
+                                    track = channel + 'edge\\*'
+                                    edge_rev = sh(returnStdout: true,
+                                        script: "snapcraft revisions $snap | " +
+                                        "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
+                                    print "edge rev is $edge_rev"
+                                    track = channel + 'beta\\*'
+                                    beta_rev = sh(returnStdout: true,
+                                        script: "snapcraft revisions $snap | " +
+                                        "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
+                                    print "beta rev is $beta_rev"
+
+                                    if (edge_rev != beta_rev) {
+                                        print "Promoting $edge_rev to beta in place of $beta_rev"
+                                        beta_track = channel + 'beta'
+                                        sh "snapcraft release $snap $edge_rev $beta_track"
+                                    }
                                 }
                             }
                         } // stage('Snap promotion')
@@ -660,34 +680,56 @@ EOF"""
             } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive)
         } // dir(OSM_DEVOPS)
     } finally {
-        if ( params.DO_INSTALL && server_id != null) {
-            delete_vm = true
-            if (error && params.SAVE_CONTAINER_ON_FAIL ) {
-                delete_vm = false
-            }
-            if (!error && params.SAVE_CONTAINER_ON_PASS ) {
-                delete_vm = false
-            }
+       stage('Archive Container Logs') {
+            if ( ARCHIVE_LOGS_FLAG ) {
+                try {
+                    // Archive logs
+                    remote = [
+                        name: containerName,
+                        host: IP_ADDRESS,
+                        user: 'ubuntu',
+                        identityFile: SSH_KEY,
+                        allowAnyHosts: true,
+                        logLevel: 'INFO',
+                        pty: true
+                    ]
+                    println('Archiving container logs')
+                    archive_logs(remote)
+                } catch (Exception e) {
+                    println('Error fetching logs: '+ e.getMessage())
+                }
+            } // end if ( ARCHIVE_LOGS_FLAG )
+        }
+        stage('Cleanup') {
+            if ( params.DO_INSTALL && server_id != null) {
+                delete_vm = true
+                if (error && params.SAVE_CONTAINER_ON_FAIL ) {
+                    delete_vm = false
+                }
+                if (!error && params.SAVE_CONTAINER_ON_PASS ) {
+                    delete_vm = false
+                }
 
-            if ( delete_vm ) {
-                if (server_id != null) {
-                    println("Deleting VM: $server_id")
-                    sh """#!/bin/sh -e
-                        for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done
-                        openstack server delete ${server_id}
-                    """
-                } else {
-                    println("Saved VM $server_id in ETSI VIM")
+                if ( delete_vm ) {
+                    if (server_id != null) {
+                        println("Deleting VM: $server_id")
+                        sh """#!/bin/sh -e
+                            for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done
+                            openstack server delete ${server_id}
+                        """
+                    } else {
+                        println("Saved VM $server_id in ETSI VIM")
+                    }
                 }
             }
-        }
-        if ( http_server_name != null ) {
-            sh "docker stop ${http_server_name} || true"
-            sh "docker rm ${http_server_name} || true"
-        }
+            if ( http_server_name != null ) {
+                sh "docker stop ${http_server_name} || true"
+                sh "docker rm ${http_server_name} || true"
+            }
 
-        if ( devopstempdir != null ) {
-            sh "rm -rf ${devopstempdir}"
+            if ( devopstempdir != null ) {
+                sh "rm -rf ${devopstempdir}"
+            }
         }
     }
 }