From c880c384873d058420d2e81c4a992715fcf03a3b Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Wed, 7 Sep 2022 09:36:43 -0400 Subject: [PATCH] Feature 10953: Publication of Charms in CI/CD Pipeline Adds section to detect changes in charms, and builds and publishes the charms accordingly Change-Id: I41bbcdcc0cf919ddbbc83374ccfb3f17c6b7dc37 Signed-off-by: Mark Beierl --- jenkins/ci-pipelines/ci_stage_2.groovy | 93 +++++++++++++++++++++++++- jenkins/ci-pipelines/ci_stage_3.groovy | 62 ++++++++++++++++- 2 files changed, 150 insertions(+), 5 deletions(-) diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index dc155a19..1c8911f2 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -99,8 +99,8 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } - if (fileExists('snap/snapcraft.yaml')) { - stage('Snap build') { + stage('Snap build') { + if (fileExists('snap/snapcraft.yaml')) { withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap" sh "sudo snapcraft clean --use-lxd" @@ -125,6 +125,95 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } + stage('Charm build') { + charms = [ + 'installers/charm/bundles/osm', + 'installers/charm/bundles/osm-ha', + 'installers/charm/grafana', + 'installers/charm/juju-simplestreams-operator', + 'installers/charm/kafka-exporter', + 'installers/charm/mariadb-k8s', + 'installers/charm/mongodb-exporter', + 'installers/charm/mysqld-exporter', + 'installers/charm/osm-lcm', + 'installers/charm/osm-mon', + 'installers/charm/osm-nbi', + 'installers/charm/osm-ng-ui', + 'installers/charm/osm-pol', + 'installers/charm/osm-ro', + 'installers/charm/prometheus', + 'installers/charm/vca-integrator-operator', + ] + parallelSteps = [:] + concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int + for (charmPath in charms) { + def directory = charmPath + CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l" + charmsChanged = sh(returnStdout:true, script: CHECK_CHANGES).trim() + if (charmsChanged != "0") { + println("$directory has changes, rebuilding") + + parallelSteps[directory] = { + waitUntil(quiet: true, initialRecurrencePeriod: 5) { + if (concurrency > 0) { + concurrency-- + true + } else { + sleep 5 + false + } + } + try { + dir("${directory}") { + try { + sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log" + } catch (Exception e) { + sh "cat pack-stderr.log" + throw e + } + + def resourceName = sh( + returnStdout: true, + script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim() + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + commandOutput = sh(returnStdout:true, script: "charmcraft whoami") + try { + println("Uploading $resourceName") + sh "charmcraft upload --format json --verbosity brief $resourceName 1> upload.json 2> upload-stderr.log" + } catch (Exception e) { + sh "cat upload-stderr.log" + throw e + } + def charmRevision = sh( + returnStdout: true, + script: 'cat upload.json | jq -r ".revision"').trim() + + def charmBranch="merged" + if ( !JOB_NAME.contains('merge') ) { + charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-') + } + def channel="latest" + if (BRANCH_NAME.startsWith("v")) { + channel=BRANCH_NAME.substring(1) + } else if (BRANCH_NAME!="master") { + REV="/"+BRANCH_NAME+REV.replaceAll('/','-') + } + + def charmName = sh( + returnStdout:true, + script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim() + sh "charmcraft release ${charmName} --revision=${charmRevision} --channel=${channel}/edge/${charmBranch}" + } + } + } finally { + concurrency++ + } + } + } + } + parallel parallelSteps + } if ( do_stage_3 ) { stage('Build System') { def downstream_params_stage_3 = [ diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index b6efe3c6..b052b9c7 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -638,7 +638,6 @@ EOF""" } parallel parallelSteps } - stage('Snap promotion') { withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { snaps = ['osmclient'] @@ -653,12 +652,12 @@ EOF""" edge_rev = sh(returnStdout: true, script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "edge rev is $edge_rev" track = channel + 'beta\\*' beta_rev = sh(returnStdout: true, script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "beta rev is $beta_rev" + + print "Edge: $edge_rev, Beta: $beta_rev" if (edge_rev != beta_rev) { print "Promoting $edge_rev to beta in place of $beta_rev" @@ -668,6 +667,63 @@ EOF""" } } } // stage('Snap promotion') + stage('Charm promotion') { + charms = [ + 'osm', // bundle + 'osm-ha', // bundle + 'osm-grafana', + //'juju-simplestreams-operator', + //'kafka-exporter', + 'osm-mariadb', + //'mongodb-exporter', + //'mysqld-exporter', + 'osm-lcm', + 'osm-mon', + 'osm-nbi', + 'osm-ng-ui', + 'osm-pol', + 'osm-ro', + 'osm-prometheus', + //'osm-vca-integrator-operator', + ] + for (charm in charms) { + + channel = 'latest' + if (BRANCH_NAME.startsWith('v')) { + channel = BRANCH_NAME.substring(1) + } else if (BRANCH_NAME != 'master') { + channel += '/' + BRANCH_NAME.replaceAll('/', '-') + } + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + sh "charmcraft status $charm --format json > ${charm}.json" + isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int + if (isCharm) { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/edge/merged\")| .version'" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/beta\")| .version'" + } else { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/edge/merged\")| .version'" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/beta\")| .version'" + } + println(jqScriptEdge) + println(jqScriptBeta) + // edge/merged is used in place of /edge as 10.1.0 LTS uses latest/edge + edge_rev = sh(returnStdout: true, script: jqScriptEdge).trim() + beta_rev = sh(returnStdout: true, script: jqScriptBeta).trim() + try { edge_rev = edge_rev as int } catch (NumberFormatException nfe) {edge_rev = 0} + try { beta_rev = beta_rev as int } catch (NumberFormatException nfe) {beta_rev = 0} + + print "Edge: $edge_rev, Beta: $beta_rev" + + if (edge_rev > beta_rev) { + print "Promoting $edge_rev to beta in place of $beta_rev" + beta_track = channel + 'beta' + sh "charmcraft release ${charm} --revision=${edge_rev} --channel=${channel}/beta" + } + + } + } + } // stage('Charm promotion') } // if (params.DO_DOCKERPUSH) } // stage('Archive') } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) -- 2.25.1