Adding charm build
Cherry pick of charm publication code from master. Changes
bundle to use 12.0 track instead of latest
Change-Id: I7d72f1b70ae9849dc8c3d74ca59746b259eeb58e
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
diff --git a/installers/charm/bundles/osm-ha/bundle.yaml b/installers/charm/bundles/osm-ha/bundle.yaml
index ed240fd..11e4d01 100644
--- a/installers/charm/bundles/osm-ha/bundle.yaml
+++ b/installers/charm/bundles/osm-ha/bundle.yaml
@@ -74,7 +74,7 @@
gui-y: 0
nbi:
charm: osm-nbi
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
options:
@@ -88,7 +88,7 @@
gui-y: -250
ro:
charm: osm-ro
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
options:
@@ -100,7 +100,7 @@
gui-y: 250
ng-ui:
charm: osm-ng-ui
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
resources:
@@ -110,7 +110,7 @@
gui-y: 0
lcm:
charm: osm-lcm
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
options:
@@ -123,7 +123,7 @@
gui-y: 0
mon:
charm: osm-mon
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -137,7 +137,7 @@
gui-y: 0
pol:
charm: osm-pol
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
options:
@@ -149,7 +149,7 @@
gui-y: 500
pla:
charm: osm-pla
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
options:
@@ -161,7 +161,7 @@
gui-y: -250
prometheus:
charm: osm-prometheus
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
storage:
@@ -173,7 +173,7 @@
gui-y: 250
grafana:
charm: osm-grafana
- channel: latest/stable
+ channel: 12.0/stable
scale: 3
series: kubernetes
annotations:
@@ -181,7 +181,7 @@
gui-y: 500
keystone:
charm: osm-keystone
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
resources:
keystone-image: opensourcemano/keystone:12
diff --git a/installers/charm/bundles/osm/bundle.yaml b/installers/charm/bundles/osm/bundle.yaml
index 74eefe6..78f3d4a 100644
--- a/installers/charm/bundles/osm/bundle.yaml
+++ b/installers/charm/bundles/osm/bundle.yaml
@@ -72,7 +72,7 @@
gui-y: 0
nbi:
charm: osm-nbi
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -86,7 +86,7 @@
gui-y: -250
ro:
charm: osm-ro
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -98,7 +98,7 @@
gui-y: 250
ng-ui:
charm: osm-ng-ui
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
resources:
@@ -108,7 +108,7 @@
gui-y: 0
lcm:
charm: osm-lcm
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -121,7 +121,7 @@
gui-y: 0
mon:
charm: osm-mon
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -135,7 +135,7 @@
gui-y: 0
pol:
charm: osm-pol
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -147,7 +147,7 @@
gui-y: 500
pla:
charm: osm-pla
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
options:
@@ -159,7 +159,7 @@
gui-y: -250
prometheus:
charm: osm-prometheus
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
storage:
@@ -171,7 +171,7 @@
gui-y: 250
grafana:
charm: osm-grafana
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
series: kubernetes
annotations:
@@ -179,7 +179,7 @@
gui-y: 500
keystone:
charm: osm-keystone
- channel: latest/stable
+ channel: 12.0/stable
scale: 1
resources:
keystone-image: opensourcemano/keystone:12
diff --git a/installers/charm/prometheus/charmcraft.yaml b/installers/charm/prometheus/charmcraft.yaml
index babeda8..87d0463 100644
--- a/installers/charm/prometheus/charmcraft.yaml
+++ b/installers/charm/prometheus/charmcraft.yaml
@@ -35,5 +35,7 @@
parts:
charm:
build-packages:
+ - cargo
- git
- libffi-dev
+ - rustc
diff --git a/installers/charmed_install.sh b/installers/charmed_install.sh
index 527f5a8..f68f395 100755
--- a/installers/charmed_install.sh
+++ b/installers/charmed_install.sh
@@ -21,7 +21,7 @@
K8S_CLOUD_NAME="k8s-cloud"
KUBECTL="microk8s.kubectl"
MICROK8S_VERSION=1.23
-OSMCLIENT_VERSION=latest
+OSMCLIENT_VERSION=12.0
IMAGES_OVERLAY_FILE=~/.osm/images-overlay.yaml
PASSWORD_OVERLAY_FILE=~/.osm/password-overlay.yaml
PATH=/snap/bin:${PATH}
diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy
index dc155a1..4002e01 100644
--- a/jenkins/ci-pipelines/ci_stage_2.groovy
+++ b/jenkins/ci-pipelines/ci_stage_2.groovy
@@ -31,6 +31,7 @@
def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifactory_server,docker_args="",do_stage_4=false) {
println("do_stage_3= ${do_stage_3}")
ci_helper = load "devops/jenkins/ci-pipelines/ci_helper.groovy"
+ def isMergeJob = JOB_NAME.contains('merge')
stage('Prepare') {
sh 'env'
@@ -41,7 +42,7 @@
}
stage('License Scan') {
- if (!JOB_NAME.contains('merge')) {
+ if (!isMergeJob) {
sh "devops/tools/license_scan.sh"
}
else {
@@ -51,7 +52,7 @@
stage('Release Note Check') {
if (fileExists('devops-stages/stage-releasenote.sh')) {
- if (!JOB_NAME.contains('merge')) {
+ if (!isMergeJob) {
sh "devops-stages/stage-releasenote.sh"
}
else {
@@ -99,8 +100,8 @@
}
}
- if (fileExists('snap/snapcraft.yaml')) {
- stage('Snap build') {
+ stage('Snap build') {
+ if (fileExists('snap/snapcraft.yaml')) {
withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap"
sh "sudo snapcraft clean --use-lxd"
@@ -109,7 +110,7 @@
sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/"
REV=""
- if ( !JOB_NAME.contains('merge') ) {
+ if ( ! isMergeJob ) {
REV="/"+"${GERRIT_REFSPEC}".replaceAll('/','-')
}
channel="latest"
@@ -125,6 +126,119 @@
}
}
+ stage('Charm build') {
+ charms = [
+ 'installers/charm/bundles/osm',
+ 'installers/charm/bundles/osm-ha',
+ 'installers/charm/grafana',
+ 'installers/charm/osm-lcm',
+ 'installers/charm/osm-mon',
+ 'installers/charm/osm-nbi',
+ 'installers/charm/osm-ng-ui',
+ 'installers/charm/osm-pol',
+ 'installers/charm/osm-ro',
+ 'installers/charm/prometheus',
+ ]
+ parallelSteps = [:]
+ concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int
+ for (charmPath in charms) {
+ def directory = charmPath
+ if (fileExists("$charmPath/charmcraft.yaml")) {
+ CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l"
+ charmsChanged = sh(returnStdout:true, script: CHECK_CHANGES).trim()
+ if (charmsChanged != "0" || isMergeJob) {
+ println("$directory has changes, rebuilding")
+
+ parallelSteps[directory] = {
+ waitUntil(quiet: true, initialRecurrencePeriod: 5) {
+ if (concurrency > 0) {
+ concurrency--
+ true
+ } else {
+ sleep 5
+ false
+ }
+ }
+ try {
+ dir("${directory}") {
+ try {
+ sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log"
+ } catch (Exception e) {
+ sh "cat pack-stderr.log"
+ throw e
+ }
+
+ def charmFilename = sh(
+ returnStdout: true,
+ script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim()
+
+ withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) {
+ try {
+ println("Uploading $charmFilename")
+ sh "charmcraft upload --format json --verbosity brief $charmFilename 1> upload.json 2> upload-stderr.log"
+ } catch (Exception e) {
+ def errorMessage = sh(
+ returnStdout: true,
+ script: 'cat upload.json | jq -r ".errors[0].message" | grep -v null').trim()
+ if (errorMessage.contains("already exists")) {
+ println("No changes, skipping upload")
+ return 0
+ }
+ sh "cat upload-stderr.log"
+ throw e
+ }
+ def charmRevision = sh(
+ returnStdout: true,
+ script: 'cat upload.json | jq -r ".revision"').trim()
+
+ def charmBranch="merged"
+ if ( !JOB_NAME.contains('merge') ) {
+ charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-')
+ }
+ def channel="latest"
+ if (BRANCH_NAME.startsWith("v")) {
+ channel=BRANCH_NAME.substring(1)
+ } else if (BRANCH_NAME!="master") {
+ REV="/"+BRANCH_NAME+REV.replaceAll('/','-')
+ }
+
+ def charmName = sh(
+ returnStdout:true,
+ script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim()
+
+ def resourceArgument = ""
+ try {
+ def resourceName = sh(
+ returnStdout:true,
+ script: "cat metadata.yaml | yq '.resources | keys | .[0]'").trim()
+
+ resourceArgument = "--resource $resourceName:1"
+
+ resourceName = sh(
+ returnStdout:true,
+ script: "cat metadata.yaml | yq '.resources | keys | .[1]'").trim()
+ if( resourceName != "null" ) {
+ resourceArgument += " --resource $resourceName:1"
+ }
+
+ } catch (Exception discard) {
+ print("No resource associated with bundle, skipping")
+ }
+
+ print("resourceArgument = $resourceArgument")
+
+ sh "charmcraft release ${charmName} --revision=${charmRevision} ${resourceArgument} --channel=${channel}/edge/${charmBranch}"
+ }
+ }
+ } finally {
+ concurrency++
+ }
+ }
+ }
+ }
+ }
+ parallel parallelSteps
+ }
if ( do_stage_3 ) {
stage('Build System') {
def downstream_params_stage_3 = [
diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy
index c8bcd1e..64584ee 100644
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -638,7 +638,6 @@
}
parallel parallelSteps
}
-
stage('Snap promotion') {
withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
snaps = ['osmclient']
@@ -653,12 +652,12 @@
edge_rev = sh(returnStdout: true,
script: "snapcraft revisions $snap | " +
"grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
- print "edge rev is $edge_rev"
track = channel + 'beta\\*'
beta_rev = sh(returnStdout: true,
script: "snapcraft revisions $snap | " +
"grep \"$track\" | tail -1 | awk '{print \$1}'").trim()
- print "beta rev is $beta_rev"
+
+ print "Edge: $edge_rev, Beta: $beta_rev"
if (edge_rev != beta_rev) {
print "Promoting $edge_rev to beta in place of $beta_rev"
@@ -668,6 +667,56 @@
}
}
} // stage('Snap promotion')
+ stage('Charm promotion') {
+ charms = [
+ 'osm', // bundle
+ 'osm-ha', // bundle
+ 'osm-grafana',
+ 'osm-mariadb',
+ 'osm-lcm',
+ 'osm-mon',
+ 'osm-nbi',
+ 'osm-ng-ui',
+ 'osm-pol',
+ 'osm-ro',
+ 'osm-prometheus',
+ ]
+ for (charm in charms) {
+
+ channel = 'latest'
+ if (BRANCH_NAME.startsWith('v')) {
+ channel = BRANCH_NAME.substring(1)
+ } else if (BRANCH_NAME != 'master') {
+ channel += '/' + BRANCH_NAME.replaceAll('/', '-')
+ }
+
+ withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) {
+ sh "charmcraft status $charm --format json > ${charm}.json"
+ isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int
+ if (isCharm) {
+ jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1"
+ jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"latest/beta\")| .version'|head -1"
+ } else {
+ jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/edge/merged\")| .version'|head -1"
+ jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[].releases[] | select(.channel==\"latest/beta\")| .version'|head -1"
+ }
+ // edge/merged is used in place of /edge as 10.1.0 LTS uses latest/edge
+ edge_rev = sh(returnStdout: true, script: jqScriptEdge).trim()
+ beta_rev = sh(returnStdout: true, script: jqScriptBeta).trim()
+ try { edge_rev = edge_rev as int } catch (NumberFormatException nfe) {edge_rev = 0}
+ try { beta_rev = beta_rev as int } catch (NumberFormatException nfe) {beta_rev = 0}
+
+ print "Edge: $edge_rev, Beta: $beta_rev"
+
+ if (edge_rev > beta_rev) {
+ print "Promoting $edge_rev to beta in place of $beta_rev"
+ beta_track = channel + 'beta'
+ sh "charmcraft release ${charm} --revision=${edge_rev} --channel=${channel}/beta"
+ }
+
+ }
+ }
+ } // stage('Charm promotion')
} // if (params.DO_DOCKERPUSH)
} // stage('Archive')
} // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive)
@@ -675,18 +724,22 @@
} finally {
stage('Archive Container Logs') {
if ( ARCHIVE_LOGS_FLAG ) {
- // Archive logs
- remote = [
- name: containerName,
- host: IP_ADDRESS,
- user: 'ubuntu',
- identityFile: SSH_KEY,
- allowAnyHosts: true,
- logLevel: 'INFO',
- pty: true
- ]
- println('Archiving container logs')
- archive_logs(remote)
+ try {
+ // Archive logs
+ remote = [
+ name: containerName,
+ host: IP_ADDRESS,
+ user: 'ubuntu',
+ identityFile: SSH_KEY,
+ allowAnyHosts: true,
+ logLevel: 'INFO',
+ pty: true
+ ]
+ println('Archiving container logs')
+ archive_logs(remote)
+ } catch (Exception e) {
+ println('Error fetching logs: '+ e.getMessage())
+ }
} // end if ( ARCHIVE_LOGS_FLAG )
}
stage('Cleanup') {