X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=jenkins%2Fci-pipelines%2Fci_stage_2.groovy;h=13714b9091c4a0c25c83b126a4e808b2c8f29231;hb=80b2e177597d4d66afa568837aa58e0064f1a9d5;hp=4d23cf1759129bbd2538d3c21edf2cde516683fc;hpb=27556780aaf744ac57f2a023f10009230a61316b;p=osm%2Fdevops.git diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index 4d23cf17..13714b90 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -19,17 +19,19 @@ def project_checkout(url_prefix,project,refspec,revision) { // checkout the project // this is done automaticaly by the multibranch pipeline plugin // git url: "${url_prefix}/${project}" - + sh "git fetch --tags" sh "git fetch origin ${refspec}" if (GERRIT_PATCHSET_REVISION.size() > 0 ) { sh "git checkout -f ${revision}" } + sh "sudo git clean -dfx" } def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifactory_server,docker_args="",do_stage_4=false) { println("do_stage_3= ${do_stage_3}") ci_helper = load "devops/jenkins/ci-pipelines/ci_helper.groovy" + def isMergeJob = JOB_NAME.contains('merge') stage('Prepare') { sh 'env' @@ -40,27 +42,44 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } stage('License Scan') { - if (!JOB_NAME.contains('merge')) { + if (!isMergeJob) { sh "devops/tools/license_scan.sh" - } + } else { println("skip the scan for merge") } } + stage('Release Note Check') { + if (fileExists('devops-stages/stage-releasenote.sh')) { + if (!isMergeJob) { + sh "devops-stages/stage-releasenote.sh" + } + else { + println("Not checking release notes for merge job") + } + } + else { + println("No releasenote check present") + } + } + container_name = "${project}-${branch}".toLowerCase() stage('Docker-Build') { - sh ''' - echo RUN groupadd -o -g $(id -g) -r jenkins >> Dockerfile - echo RUN useradd -o -u $(id -u) --create-home -r -g jenkins jenkins >> Dockerfile - ''' - sh "docker build -t ${container_name} ." + APT_PROXY = "http://172.21.1.1:3142" + sh "docker build --build-arg APT_PROXY=${APT_PROXY} -t ${container_name} ." } - withDockerContainer(image: "${container_name}", args: docker_args) { + UID = sh(returnStdout:true, script: 'id -u').trim() + GID = sh(returnStdout:true, script: 'id -g').trim() + withDockerContainer(image: "${container_name}", args: docker_args + " -u root") { stage('Test') { - sh 'devops-stages/stage-test.sh' + sh "groupadd -o -g $GID -r jenkins" + sh "useradd -o -u $UID -d `pwd` -r -g jenkins jenkins" + sh "echo '#! /bin/sh' > /usr/bin/mesg" + sh "chmod 755 /usr/bin/mesg" + sh "runuser jenkins -c devops-stages/stage-test.sh" if (fileExists('coverage.xml')) { cobertura coberturaReportFile: 'coverage.xml' } @@ -69,21 +88,185 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } stage('Build') { - sh(returnStdout:true, script: 'devops-stages/stage-build.sh').trim() + sh(returnStdout:true, + script: "runuser jenkins -c devops-stages/stage-build.sh").trim() } stage('Archive') { - sh "mkdir -p changelog" - sh "devops/tools/generatechangelog-pipeline.sh > changelog/changelog-${mdg}.html" - sh(returnStdout:true, script: 'devops-stages/stage-archive.sh').trim() + sh "runuser jenkins -c 'mkdir -p changelog'" + sh "runuser jenkins -c \"devops/tools/generatechangelog-pipeline.sh > changelog/changelog-${mdg}.html\"" + sh(returnStdout:true, + script: "runuser jenkins -c devops-stages/stage-archive.sh").trim() ci_helper.archive(artifactory_server,mdg,branch,'untested') } } - if ( do_stage_3 ) { + stage('Snap build') { + if (fileExists('snap/snapcraft.yaml')) { + withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) { + sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap" + sh "sudo snapcraft clean --use-lxd" + sh "snapcraft --use-lxd" + sh "mv ${WORKSPACE}/${mdg}_*.snap ${WORKSPACE}/${mdg}.snap" + sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/" + + REV="" + if ( ! isMergeJob ) { + REV="/"+"${GERRIT_REFSPEC}".replaceAll('/','-') + } + channel="latest" + if (BRANCH_NAME.startsWith("v")) { + channel=BRANCH_NAME.substring(1) + } else if (BRANCH_NAME!="master") { + REV="/"+BRANCH_NAME+REV.replaceAll('/','-') + } + + sh "snapcraft push --release=${channel}/edge${REV} ${mdg}.snap" + sh "sudo rm -rf ${WORKSPACE}/*.snap" + } + } + } + + stage('Charm build') { + charms = [ + 'installers/charm/bundles/osm', + 'installers/charm/bundles/osm-ha', + 'installers/charm/grafana', + 'installers/charm/mongodb-exporter', + 'installers/charm/mysqld-exporter', + 'installers/charm/osm-lcm', + 'installers/charm/osm-mon', + 'installers/charm/osm-nbi', + 'installers/charm/osm-ng-ui', + 'installers/charm/osm-pol', + 'installers/charm/osm-ro', + 'installers/charm/osm-update-db-operator', + 'installers/charm/prometheus', + 'installers/charm/vca-integrator-operator', + ] + parallelSteps = [:] + concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int + for (charmPath in charms) { + def directory = charmPath + if (fileExists("$charmPath/charmcraft.yaml")) { + if (isMergeJob) { + CHECK_CHANGES = "git diff --name-only HEAD^1 -- $directory |wc -l" + } else { + CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l" + } + charmsChanged = sh(returnStdout:true, script: CHECK_CHANGES).trim() + + if (charmsChanged != "0") { + println("$directory has changes, rebuilding") + parallelSteps[directory] = { + waitUntil(quiet: true, initialRecurrencePeriod: 5) { + if (concurrency > 0) { + concurrency-- + true + } else { + sleep 5 + false + } + } + try { + dir("${directory}") { + try { + sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log" + } catch (Exception e) { + sh "cat pack-stderr.log" + throw e + } + + def charmFilename = sh( + returnStdout: true, + script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim() + + withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { + try { + println("Uploading $charmFilename") + sh "charmcraft upload --format json --verbosity brief $charmFilename 1> upload.json 2> upload-stderr.log" + } catch (Exception e) { + def errorMessage = sh( + returnStdout: true, + script: 'cat upload.json | jq -r ".errors[0].message" | grep -v null').trim() + if (errorMessage.contains("already exists")) { + println("No changes, skipping upload") + return 0 + } + sh "cat upload-stderr.log" + throw e + } + def charmRevision = sh( + returnStdout: true, + script: 'cat upload.json | jq -r ".revision"').trim() + + def charmBranch="merged" + if ( !JOB_NAME.contains('merge') ) { + charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-') + } + def channel="latest" + if (BRANCH_NAME.startsWith("v")) { + channel=BRANCH_NAME.substring(1) + } else if (BRANCH_NAME!="master" && JOB_NAME.contains('merge')) { + charmBranch=BRANCH_NAME + } + + def charmName = sh( + returnStdout:true, + script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim() + + def resourceArgument = "" + try { + def resourceName = sh( + returnStdout:true, + script: "cat metadata.yaml | yq '.resources | keys | .[0]'" + ).trim() + if( resourceName != "null" ) { + sh "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'" + def resourceVers = sh( + returnStdout:true, + script: "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'" + ).trim() + + resourceArgument = "--resource $resourceName:$resourceVers" + + resourceName = sh( + returnStdout:true, + script: "cat metadata.yaml | yq '.resources | keys | .[1]'").trim() + + if( resourceName != "null" ) { + resourceVers = sh( + returnStdout:true, + script: "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'" + ).trim() + resourceArgument += " --resource $resourceName:$resourceVers" + } + } + + } catch (Exception discard) { + print("No resource associated with bundle, skipping") + } + + print("resourceArgument = $resourceArgument") + + sh "charmcraft release ${charmName} --revision=${charmRevision} ${resourceArgument} --channel=${channel}/edge/${charmBranch}" + } + } + } finally { + concurrency++ + } + } + } + } + } + parallel parallelSteps + } + if ( do_stage_3 ) { stage('Build System') { def downstream_params_stage_3 = [ string(name: 'GERRIT_BRANCH', value: "${branch}"), + string(name: 'INSTALLER', value: "Default" ), + string(name: 'OPENSTACK_BASE_IMAGE', value: "ubuntu22.04" ), string(name: 'UPSTREAM_JOB_NAME', value: "${JOB_NAME}" ), string(name: 'UPSTREAM_JOB_NUMBER', value: "${BUILD_NUMBER}" ), booleanParam(name: 'DO_STAGE_4', value: do_stage_4 ) @@ -102,7 +285,6 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } } - } return this