Feature 10953: Publication of Charms in CI/CD Pipeline

Adds section to detect changes in charms, and builds
and publishes the charms accordingly

Change-Id: I41bbcdcc0cf919ddbbc83374ccfb3f17c6b7dc37
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>

diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy
index dc155a1..1c8911f 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_2.groovy
+++ b/jenkins/ci-pipelines/ci_stage_2.groovy
@@ -99,8 +99,8 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa
         }
     }
 
-    if (fileExists('snap/snapcraft.yaml')) {
-        stage('Snap build') {
+    stage('Snap build') {
+        if (fileExists('snap/snapcraft.yaml')) {
             withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
                 sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap"
                 sh "sudo snapcraft clean --use-lxd"
@@ -125,6 +125,95 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa
         }
     }
 
+    stage('Charm build') {
+        charms = [
+            'installers/charm/bundles/osm',
+            'installers/charm/bundles/osm-ha',
+            'installers/charm/grafana',
+            'installers/charm/juju-simplestreams-operator',
+            'installers/charm/kafka-exporter',
+            'installers/charm/mariadb-k8s',
+            'installers/charm/mongodb-exporter',
+            'installers/charm/mysqld-exporter',
+            'installers/charm/osm-lcm',
+            'installers/charm/osm-mon',
+            'installers/charm/osm-nbi',
+            'installers/charm/osm-ng-ui',
+            'installers/charm/osm-pol',
+            'installers/charm/osm-ro',
+            'installers/charm/prometheus',
+            'installers/charm/vca-integrator-operator',
+        ]
+        parallelSteps = [:]
+        concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int
+        for (charmPath in charms) {
+            def directory = charmPath
+            CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l"
+            charmsChanged = sh(returnStdout:true,  script: CHECK_CHANGES).trim()
+            if (charmsChanged != "0") {
+                println("$directory has changes, rebuilding")
+
+                parallelSteps[directory] = {
+                    waitUntil(quiet: true, initialRecurrencePeriod: 5) {
+                        if (concurrency > 0) {
+                            concurrency--
+                            true
+                        } else {
+                            sleep 5
+                            false
+                        }
+                    }
+                    try {
+                        dir("${directory}") {
+                            try {
+                                sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log"
+                            } catch (Exception e) {
+                                sh "cat pack-stderr.log"
+                                throw e
+                            }
+
+                            def resourceName = sh(
+                                returnStdout: true,
+                                script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim()
+
+                            withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) {
+                                commandOutput = sh(returnStdout:true, script: "charmcraft whoami")
+                                try {
+                                    println("Uploading $resourceName")
+                                    sh "charmcraft upload --format json --verbosity brief $resourceName 1> upload.json 2> upload-stderr.log"
+                                } catch (Exception e) {
+                                    sh "cat upload-stderr.log"
+                                    throw e
+                                }
+                                def charmRevision = sh(
+                                    returnStdout: true,
+                                    script: 'cat upload.json | jq -r ".revision"').trim()
+
+                                def charmBranch="merged"
+                                if ( !JOB_NAME.contains('merge') ) {
+                                    charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-')
+                                }
+                                def channel="latest"
+                                if (BRANCH_NAME.startsWith("v")) {
+                                    channel=BRANCH_NAME.substring(1)
+                                } else if (BRANCH_NAME!="master") {
+                                    REV="/"+BRANCH_NAME+REV.replaceAll('/','-')
+                                }
+
+                                def charmName = sh(
+                                    returnStdout:true,
+                                    script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim()
+                                sh "charmcraft release ${charmName} --revision=${charmRevision} --channel=${channel}/edge/${charmBranch}"
+                            }
+                        }
+                    } finally {
+                        concurrency++
+                    }
+                }
+            }
+        }
+        parallel parallelSteps
+    }
     if ( do_stage_3 ) {
         stage('Build System') {
             def downstream_params_stage_3 = [

diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy
index b6efe3c..b052b9c 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -638,7 +638,6 @@ EOF"""
                             }
                             parallel parallelSteps
                         }
-
                         stage('Snap promotion') {
                             withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
                                 snaps = ['osmclient']
@@ -653,12 +652,12 @@ EOF"""
                                     edge_rev = sh(returnStdout: true,
                                         script: "snapcraft revisions $snap | " +
                                         "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                    print "edge rev is $edge_rev"
                                     track = channel + 'beta\\*'
                                     beta_rev = sh(returnStdout: true,
                                         script: "snapcraft revisions $snap | " +
                                         "grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
-                                    print "beta rev is $beta_rev"
+
+                                    print "Edge: $edge_rev, Beta: $beta_rev"
 
                                     if (edge_rev != beta_rev) {
                                         print "Promoting $edge_rev to beta in place of $beta_rev"
@@ -668,6 +667,63 @@ EOF"""
                                 }
                             }
                         } // stage('Snap promotion')
+                        stage('Charm promotion') {
+                            charms = [
+                                'osm', // bundle
+                                'osm-ha', // bundle
+                                'osm-grafana',
+                                //'juju-simplestreams-operator',
+                                //'kafka-exporter',
+                                'osm-mariadb',
+                                //'mongodb-exporter',
+                                //'mysqld-exporter',
+                                'osm-lcm',
+                                'osm-mon',
+                                'osm-nbi',
+                                'osm-ng-ui',
+                                'osm-pol',
+                                'osm-ro',
+                                'osm-prometheus',
+                                //'osm-vca-integrator-operator',
+                            ]
+                            for (charm in charms) {
+
+                                channel = 'latest'
+                                if (BRANCH_NAME.startsWith('v')) {
+                                    channel = BRANCH_NAME.substring(1)
+                                } else if (BRANCH_NAME != 'master') {
+                                    channel += '/' + BRANCH_NAME.replaceAll('/', '-')
+                                }
+
+                                withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) {
+                                    sh "charmcraft status $charm --format json > ${charm}.json"
+                                    isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int
+                                    if (isCharm) {
+                                        jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/edge/merged\")| .version'"
+                                        jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/beta\")| .version'"
+                                    } else {
+                                        jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/edge/merged\")| .version'"
+                                        jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/beta\")| .version'"
+                                    }
+                                    println(jqScriptEdge)
+                                    println(jqScriptBeta)
+                                    // edge/merged is used in place of /edge as 10.1.0 LTS uses latest/edge
+                                    edge_rev = sh(returnStdout: true, script: jqScriptEdge).trim()
+                                    beta_rev = sh(returnStdout: true, script: jqScriptBeta).trim()
+                                    try { edge_rev = edge_rev as int } catch (NumberFormatException nfe) {edge_rev = 0}
+                                    try { beta_rev = beta_rev as int } catch (NumberFormatException nfe) {beta_rev = 0}
+
+                                    print "Edge: $edge_rev, Beta: $beta_rev"
+
+                                    if (edge_rev > beta_rev) {
+                                        print "Promoting $edge_rev to beta in place of $beta_rev"
+                                        beta_track = channel + 'beta'
+                                        sh "charmcraft release ${charm} --revision=${edge_rev} --channel=${channel}/beta"
+                                    }
+
+                                }
+                            }
+                        } // stage('Charm promotion')
                     } // if (params.DO_DOCKERPUSH)
                 } // stage('Archive')
             } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive)