Adding logging around repo

[osm/devops.git] / jenkins / ci-pipelines / ci_stage_3.groovy

diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy
index 44117ad..05d795d 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -200,7 +200,9 @@ node("${params.NODE}") {
 ///////////////////////////////////////////////////////////////////////////////////////
         stage("Copy Artifacts") {
             // cleanup any previous repo
-            sh 'rm -rf repo'
+            sh "tree -fD repo || exit 0"
+            sh 'rm -rvf repo'
+            sh "tree -fD repo && lsof repo || exit 0"
             dir("repo") {
                 packageList = []
                 dir("${RELEASE}") {
@@ -294,12 +296,17 @@ node("${params.NODE}") {
                 http_server_name = "${container_name}-apache"
 
                 pwd = sh(returnStdout:true,  script: 'pwd').trim()
-                repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim()
-                repo_base_url = ci_helper.start_http_server(pwd,http_server_name,repo_port)
-                NODE_IP_ADDRESS=sh(returnStdout: true, script:
+                repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0));' +
+                               'print(s.getsockname()[1]); s.close()\');',
+                               returnStdout: true).trim()
+                internal_docker_http_server_url = ci_helper.start_http_server(pwd, http_server_name, repo_port)
+                NODE_IP_ADDRESS = sh(returnStdout: true, script:
                     "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim()
+                ci_helper.check_status_http_server(NODE_IP_ADDRESS, repo_port)
             }
 
+            sh "tree -fD repo"
+
             // Unpack devops package into temporary location so that we use it from upstream if it was part of a patch
             osm_devops_dpkg = sh(returnStdout: true, script: "find ./repo/release/pool/ -name osm-devops*.deb").trim()
             devopstempdir = sh(returnStdout: true, script: "mktemp -d").trim()
@@ -393,10 +400,16 @@ node("${params.NODE}") {
                     println("Waiting for VM at ${IP_ADDRESS} to be reachable")
 
                     alive = false
-                    while (! alive) {
-                        output=sh(returnStdout: true, script: "sleep 1 ; nc -zv ${IP_ADDRESS} 22 2>&1 || true").trim()
-                        println("output is [$output]")
-                        alive = output.contains("succeeded")
+                    timeout(time: 1, unit: 'MINUTES') {
+                        while (!alive) {
+                            output = sh(
+                                returnStatus: true,
+                                script: "ssh -T -i ${SSH_KEY} " +
+                                    "-o StrictHostKeyChecking=no " +
+                                    "-o UserKnownHostsFile=/dev/null " +
+                                    "-o ConnectTimeout=5 ubuntu@${IP_ADDRESS} 'echo Alive'")
+                            alive = (output == 0)
+                        }
                     }
                     println("VM is ready and accepting ssh connections")
                 } // stage("Spawn Remote VM")
@@ -447,6 +460,9 @@ node("${params.NODE}") {
                     remote.logLevel = 'INFO'
                     remote.pty = true
 
+                    // Ensure the VM is ready
+                    sshCommand remote: remote, command: 'cloud-init status --wait'
+
                     // Force time sync to avoid clock drift and invalid certificates
                     sshCommand remote: remote, command: """
                         sudo apt update
@@ -594,7 +610,9 @@ EOF"""
 
                                 parallelSteps[module] = {
                                     dir("$module") {
-                                        sh "docker tag opensourcemano/${moduleName}:${moduleTag} opensourcemano/${moduleName}:${dockerTag}"
+                                        sh("docker pull ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}")
+                                        sh("""docker tag opensourcemano/${moduleName}:${moduleTag} \
+                                           opensourcemano/${moduleName}:${dockerTag}""")
                                         sh "docker push opensourcemano/${moduleName}:${dockerTag}"
                                     }
                                 }
@@ -602,31 +620,32 @@ EOF"""
                             parallel parallelSteps
                         }
 
-                        stage("Snap promotion") {
-                            def snaps = ["osmclient"]
-                            sh "snapcraft login --with ~/.snapcraft/config"
-                            for (snap in snaps) {
-                                channel="latest/"
-                                if (BRANCH_NAME.startsWith("v")) {
-                                    channel=BRANCH_NAME.substring(1)+"/"
-                                } else if (BRANCH_NAME!="master") {
-                                    channel+="/"+BRANCH_NAME.replaceAll('/','-')
-                                }
-                                track=channel+"edge\\*"
-                                edge_rev=sh(returnStdout: true,
-                                    script: "snapcraft revisions $snap | " +
-                                    "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                print "edge rev is $edge_rev"
-                                track=channel+"beta\\*"
-                                beta_rev=sh(returnStdout: true,
-                                    script: "snapcraft revisions $snap | " +
-                                    "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                print "beta rev is $beta_rev"
-
-                                if ( edge_rev != beta_rev ) {
-                                    print "Promoting $edge_rev to beta in place of $beta_rev"
-                                    beta_track=channel+"beta"
-                                    sh "snapcraft release $snap $edge_rev $beta_track"
+                        stage('Snap promotion') {
+                            withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
+                                snaps = ['osmclient']
+                                for (snap in snaps) {
+                                    channel = 'latest/'
+                                    if (BRANCH_NAME.startsWith('v')) {
+                                        channel = BRANCH_NAME.substring(1) + '/'
+                                    } else if (BRANCH_NAME != 'master') {
+                                        channel += '/' + BRANCH_NAME.replaceAll('/', '-')
+                                    }
+                                    track = channel + 'edge\\*'
+                                    edge_rev = sh(returnStdout: true,
+                                        script: "snapcraft revisions $snap | " +
+                                        "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
+                                    print "edge rev is $edge_rev"
+                                    track = channel + 'beta\\*'
+                                    beta_rev = sh(returnStdout: true,
+                                        script: "snapcraft revisions $snap | " +
+                                        "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
+                                    print "beta rev is $beta_rev"
+
+                                    if (edge_rev != beta_rev) {
+                                        print "Promoting $edge_rev to beta in place of $beta_rev"
+                                        beta_track = channel + 'beta'
+                                        sh "snapcraft release $snap $edge_rev $beta_track"
+                                    }
                                 }
                             }
                         } // stage("Snap promotion")