From b6d802a7344736a558729f0908f0730357dd38a2 Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Tue, 20 Sep 2022 13:10:38 -0400 Subject: [PATCH] Adding charm build Cherry pick of charm publication code from master. Changes bundle to use 12.0 track instead of latest Change-Id: I7d72f1b70ae9849dc8c3d74ca59746b259eeb58e Signed-off-by: Mark Beierl --- installers/charm/bundles/osm-ha/bundle.yaml | 20 ++-- installers/charm/bundles/osm/bundle.yaml | 20 ++-- installers/charm/prometheus/charmcraft.yaml | 2 + installers/charmed_install.sh | 2 +- jenkins/ci-pipelines/ci_stage_2.groovy | 124 +++++++++++++++++++- jenkins/ci-pipelines/ci_stage_3.groovy | 83 ++++++++++--- 6 files changed, 210 insertions(+), 41 deletions(-) diff --git a/installers/charm/bundles/osm-ha/bundle.yaml b/installers/charm/bundles/osm-ha/bundle.yaml index ed240fd8..11e4d018 100644 --- a/installers/charm/bundles/osm-ha/bundle.yaml +++ b/installers/charm/bundles/osm-ha/bundle.yaml @@ -74,7 +74,7 @@ applications: gui-y: 0 nbi: charm: osm-nbi - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes options: @@ -88,7 +88,7 @@ applications: gui-y: -250 ro: charm: osm-ro - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes options: @@ -100,7 +100,7 @@ applications: gui-y: 250 ng-ui: charm: osm-ng-ui - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes resources: @@ -110,7 +110,7 @@ applications: gui-y: 0 lcm: charm: osm-lcm - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes options: @@ -123,7 +123,7 @@ applications: gui-y: 0 mon: charm: osm-mon - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -137,7 +137,7 @@ applications: gui-y: 0 pol: charm: osm-pol - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes options: @@ -149,7 +149,7 @@ applications: gui-y: 500 pla: charm: osm-pla - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes options: @@ -161,7 +161,7 @@ applications: gui-y: -250 prometheus: charm: osm-prometheus - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes storage: @@ -173,7 +173,7 @@ applications: gui-y: 250 grafana: charm: osm-grafana - channel: latest/stable + channel: 12.0/stable scale: 3 series: kubernetes annotations: @@ -181,7 +181,7 @@ applications: gui-y: 500 keystone: charm: osm-keystone - channel: latest/stable + channel: 12.0/stable scale: 1 resources: keystone-image: opensourcemano/keystone:12 diff --git a/installers/charm/bundles/osm/bundle.yaml b/installers/charm/bundles/osm/bundle.yaml index 74eefe60..78f3d4ae 100644 --- a/installers/charm/bundles/osm/bundle.yaml +++ b/installers/charm/bundles/osm/bundle.yaml @@ -72,7 +72,7 @@ applications: gui-y: 0 nbi: charm: osm-nbi - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -86,7 +86,7 @@ applications: gui-y: -250 ro: charm: osm-ro - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -98,7 +98,7 @@ applications: gui-y: 250 ng-ui: charm: osm-ng-ui - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes resources: @@ -108,7 +108,7 @@ applications: gui-y: 0 lcm: charm: osm-lcm - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -121,7 +121,7 @@ applications: gui-y: 0 mon: charm: osm-mon - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -135,7 +135,7 @@ applications: gui-y: 0 pol: charm: osm-pol - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -147,7 +147,7 @@ applications: gui-y: 500 pla: charm: osm-pla - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes options: @@ -159,7 +159,7 @@ applications: gui-y: -250 prometheus: charm: osm-prometheus - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes storage: @@ -171,7 +171,7 @@ applications: gui-y: 250 grafana: charm: osm-grafana - channel: latest/stable + channel: 12.0/stable scale: 1 series: kubernetes annotations: @@ -179,7 +179,7 @@ applications: gui-y: 500 keystone: charm: osm-keystone - channel: latest/stable + channel: 12.0/stable scale: 1 resources: keystone-image: opensourcemano/keystone:12 diff --git a/installers/charm/prometheus/charmcraft.yaml b/installers/charm/prometheus/charmcraft.yaml index babeda83..87d04635 100644 --- a/installers/charm/prometheus/charmcraft.yaml +++ b/installers/charm/prometheus/charmcraft.yaml @@ -35,5 +35,7 @@ bases: parts: charm: build-packages: + - cargo - git - libffi-dev + - rustc diff --git a/installers/charmed_install.sh b/installers/charmed_install.sh index 527f5a8a..f68f395c 100755 --- a/installers/charmed_install.sh +++ b/installers/charmed_install.sh @@ -21,7 +21,7 @@ JUJU_AGENT_VERSION=2.9.29 K8S_CLOUD_NAME="k8s-cloud" KUBECTL="microk8s.kubectl" MICROK8S_VERSION=1.23 -OSMCLIENT_VERSION=latest +OSMCLIENT_VERSION=12.0 IMAGES_OVERLAY_FILE=~/.osm/images-overlay.yaml PASSWORD_OVERLAY_FILE=~/.osm/password-overlay.yaml PATH=/snap/bin:${PATH} diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index dc155a19..4002e010 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -31,6 +31,7 @@ def project_checkout(url_prefix,project,refspec,revision) { def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifactory_server,docker_args="",do_stage_4=false) { println("do_stage_3= ${do_stage_3}") ci_helper = load "devops/jenkins/ci-pipelines/ci_helper.groovy" + def isMergeJob = JOB_NAME.contains('merge') stage('Prepare') { sh 'env' @@ -41,7 +42,7 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } stage('License Scan') { - if (!JOB_NAME.contains('merge')) { + if (!isMergeJob) { sh "devops/tools/license_scan.sh" } else { @@ -51,7 +52,7 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa stage('Release Note Check') { if (fileExists('devops-stages/stage-releasenote.sh')) { - if (!JOB_NAME.contains('merge')) { + if (!isMergeJob) { sh "devops-stages/stage-releasenote.sh" } else { @@ -99,8 +100,8 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } - if (fileExists('snap/snapcraft.yaml')) { - stage('Snap build') { + stage('Snap build') { + if (fileExists('snap/snapcraft.yaml')) { withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap" sh "sudo snapcraft clean --use-lxd" @@ -109,7 +110,7 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/" REV="" - if ( !JOB_NAME.contains('merge') ) { + if ( ! isMergeJob ) { REV="/"+"${GERRIT_REFSPEC}".replaceAll('/','-') } channel="latest" @@ -125,6 +126,119 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } + stage('Charm build') { + charms = [ + 'installers/charm/bundles/osm', + 'installers/charm/bundles/osm-ha', + 'installers/charm/grafana', + 'installers/charm/osm-lcm', + 'installers/charm/osm-mon', + 'installers/charm/osm-nbi', + 'installers/charm/osm-ng-ui', + 'installers/charm/osm-pol', + 'installers/charm/osm-ro', + 'installers/charm/prometheus', + ] + parallelSteps = [:] + concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int + for (charmPath in charms) { + def directory = charmPath + if (fileExists("$charmPath/charmcraft.yaml")) { + CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l" + charmsChanged = sh(returnStdout:true, script: CHECK_CHANGES).trim() + if (charmsChanged != "0" || isMergeJob) { + println("$directory has changes, rebuilding") + + parallelSteps[directory] = { + waitUntil(quiet: true, initialRecurrencePeriod: 5) { + if (concurrency > 0) { + concurrency-- + true + } else { + sleep 5 + false + } + } + try { + dir("${directory}") { + try { + sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log" + } catch (Exception e) { + sh "cat pack-stderr.log" + throw e + } + + def charmFilename = sh( + returnStdout: true, + script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim() + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + try { + println("Uploading $charmFilename") + sh "charmcraft upload --format json --verbosity brief $charmFilename 1> upload.json 2> upload-stderr.log" + } catch (Exception e) { + def errorMessage = sh( + returnStdout: true, + script: 'cat upload.json | jq -r ".errors[0].message" | grep -v null').trim() + if (errorMessage.contains("already exists")) { + println("No changes, skipping upload") + return 0 + } + sh "cat upload-stderr.log" + throw e + } + def charmRevision = sh( + returnStdout: true, + script: 'cat upload.json | jq -r ".revision"').trim() + + def charmBranch="merged" + if ( !JOB_NAME.contains('merge') ) { + charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-') + } + def channel="latest" + if (BRANCH_NAME.startsWith("v")) { + channel=BRANCH_NAME.substring(1) + } else if (BRANCH_NAME!="master") { + REV="/"+BRANCH_NAME+REV.replaceAll('/','-') + } + + def charmName = sh( + returnStdout:true, + script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim() + + def resourceArgument = "" + try { + def resourceName = sh( + returnStdout:true, + script: "cat metadata.yaml | yq '.resources | keys | .[0]'").trim() + + resourceArgument = "--resource $resourceName:1" + + resourceName = sh( + returnStdout:true, + script: "cat metadata.yaml | yq '.resources | keys | .[1]'").trim() + if( resourceName != "null" ) { + resourceArgument += " --resource $resourceName:1" + } + + } catch (Exception discard) { + print("No resource associated with bundle, skipping") + } + + print("resourceArgument = $resourceArgument") + + sh "charmcraft release ${charmName} --revision=${charmRevision} ${resourceArgument} --channel=${channel}/edge/${charmBranch}" + } + } + } finally { + concurrency++ + } + } + } + } + } + parallel parallelSteps + } if ( do_stage_3 ) { stage('Build System') { def downstream_params_stage_3 = [ diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index c8bcd1e6..64584eef 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -638,7 +638,6 @@ EOF""" } parallel parallelSteps } - stage('Snap promotion') { withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { snaps = ['osmclient'] @@ -653,12 +652,12 @@ EOF""" edge_rev = sh(returnStdout: true, script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "edge rev is $edge_rev" track = channel + 'beta\\*' beta_rev = sh(returnStdout: true, script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() - print "beta rev is $beta_rev" + + print "Edge: $edge_rev, Beta: $beta_rev" if (edge_rev != beta_rev) { print "Promoting $edge_rev to beta in place of $beta_rev" @@ -668,6 +667,56 @@ EOF""" } } } // stage('Snap promotion') + stage('Charm promotion') { + charms = [ + 'osm', // bundle + 'osm-ha', // bundle + 'osm-grafana', + 'osm-mariadb', + 'osm-lcm', + 'osm-mon', + 'osm-nbi', + 'osm-ng-ui', + 'osm-pol', + 'osm-ro', + 'osm-prometheus', + ] + for (charm in charms) { + + channel = 'latest' + if (BRANCH_NAME.startsWith('v')) { + channel = BRANCH_NAME.substring(1) + } else if (BRANCH_NAME != 'master') { + channel += '/' + BRANCH_NAME.replaceAll('/', '-') + } + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + sh "charmcraft status $charm --format json > ${charm}.json" + isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int + if (isCharm) { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/beta\")| .version'|head -1" + } else { + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/beta\")| .version'|head -1" + } + // edge/merged is used in place of /edge as 10.1.0 LTS uses latest/edge + edge_rev = sh(returnStdout: true, script: jqScriptEdge).trim() + beta_rev = sh(returnStdout: true, script: jqScriptBeta).trim() + try { edge_rev = edge_rev as int } catch (NumberFormatException nfe) {edge_rev = 0} + try { beta_rev = beta_rev as int } catch (NumberFormatException nfe) {beta_rev = 0} + + print "Edge: $edge_rev, Beta: $beta_rev" + + if (edge_rev > beta_rev) { + print "Promoting $edge_rev to beta in place of $beta_rev" + beta_track = channel + 'beta' + sh "charmcraft release ${charm} --revision=${edge_rev} --channel=${channel}/beta" + } + + } + } + } // stage('Charm promotion') } // if (params.DO_DOCKERPUSH) } // stage('Archive') } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) @@ -675,18 +724,22 @@ EOF""" } finally { stage('Archive Container Logs') { if ( ARCHIVE_LOGS_FLAG ) { - // Archive logs - remote = [ - name: containerName, - host: IP_ADDRESS, - user: 'ubuntu', - identityFile: SSH_KEY, - allowAnyHosts: true, - logLevel: 'INFO', - pty: true - ] - println('Archiving container logs') - archive_logs(remote) + try { + // Archive logs + remote = [ + name: containerName, + host: IP_ADDRESS, + user: 'ubuntu', + identityFile: SSH_KEY, + allowAnyHosts: true, + logLevel: 'INFO', + pty: true + ] + println('Archiving container logs') + archive_logs(remote) + } catch (Exception e) { + println('Error fetching logs: '+ e.getMessage()) + } } // end if ( ARCHIVE_LOGS_FLAG ) } stage('Cleanup') { -- 2.25.1