jenkins/ci-pipelines/ci_stage_2.groovy

   1 /* Copyright 2017 Sandvine
   2  *
   3  * All Rights Reserved.
   4  *
   5  *   Licensed under the Apache License, Version 2.0 (the "License"); you may
   6  *   not use this file except in compliance with the License. You may obtain
   7  *   a copy of the License at
   8  *
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *   Unless required by applicable law or agreed to in writing, software
  12  *   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  *   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  *   License for the specific language governing permissions and limitations
  15  *   under the License.
  16  */
  17
  18 def project_checkout(url_prefix,project,refspec,revision) {
  19     // checkout the project
  20     // this is done automaticaly by the multibranch pipeline plugin
  21     // git url: "${url_prefix}/${project}"
  22
  23     sh "git fetch --tags"
  24     sh "git fetch origin ${refspec}"
  25     if (GERRIT_PATCHSET_REVISION.size() > 0 ) {
  26         sh "git checkout -f ${revision}"
  27     }
  28     sh "sudo git clean -dfx"
  29 }
  30
  31 def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifactory_server,docker_args="",do_stage_4=false) {
  32     println("do_stage_3= ${do_stage_3}")
  33     ci_helper = load "devops/jenkins/ci-pipelines/ci_helper.groovy"
  34     def isMergeJob = JOB_NAME.contains('merge')
  35
  36     stage('Prepare') {
  37         sh 'env'
  38     }
  39
  40     stage('Checkout') {
  41         project_checkout(url_prefix,project,refspec,revision)
  42     }
  43
  44     stage('License Scan') {
  45       if (!isMergeJob) {
  46         sh "devops/tools/license_scan.sh"
  47       }
  48       else {
  49         println("skip the scan for merge")
  50       }
  51     }
  52
  53     stage('Release Note Check') {
  54       if (fileExists('devops-stages/stage-releasenote.sh')) {
  55         if (!isMergeJob) {
  56             sh "devops-stages/stage-releasenote.sh"
  57         }
  58         else {
  59             println("Not checking release notes for merge job")
  60         }
  61       }
  62       else {
  63           println("No releasenote check present")
  64       }
  65     }
  66
  67     container_name = "${project}-${branch}".toLowerCase()
  68
  69     stage('Docker-Build') {
  70         APT_PROXY = "http://172.21.1.1:3142"
  71         sh "docker build --build-arg APT_PROXY=${APT_PROXY} -t ${container_name} ."
  72     }
  73
  74     UID = sh(returnStdout:true,  script: 'id -u').trim()
  75     GID = sh(returnStdout:true,  script: 'id -g').trim()
  76     withDockerContainer(image: "${container_name}", args: docker_args + " -u root") {
  77         stage('Test') {
  78             sh "groupadd -o -g $GID -r jenkins"
  79             sh "useradd -o -u $UID -d `pwd` -r -g jenkins jenkins"
  80             sh "echo '#! /bin/sh' > /usr/bin/mesg"
  81             sh "chmod 755 /usr/bin/mesg"
  82             sh "runuser jenkins -c devops-stages/stage-test.sh"
  83             if (fileExists('coverage.xml')) {
  84                 cobertura coberturaReportFile: 'coverage.xml'
  85             }
  86             if (fileExists('nosetests.xml')) {
  87                 junit 'nosetests.xml'
  88             }
  89         }
  90         stage('Build') {
  91             sh(returnStdout:true,
  92                 script: "runuser jenkins -c devops-stages/stage-build.sh").trim()
  93         }
  94         stage('Archive') {
  95             sh "runuser jenkins -c 'mkdir -p changelog'"
  96             sh "runuser jenkins -c \"devops/tools/generatechangelog-pipeline.sh > changelog/changelog-${mdg}.html\""
  97             sh(returnStdout:true,
  98                 script: "runuser jenkins -c devops-stages/stage-archive.sh").trim()
  99             ci_helper.archive(artifactory_server,mdg,branch,'untested')
 100         }
 101     }
 102
 103     stage('Snap build') {
 104         if (fileExists('snap/snapcraft.yaml')) {
 105             withCredentials([string(credentialsId: 'Snapstore', variable: 'SNAPCRAFT_STORE_CREDENTIALS')]) {
 106                 sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/ ${WORKSPACE}/*.snap"
 107                 sh "sudo snapcraft clean --use-lxd"
 108                 sh "snapcraft --use-lxd"
 109                 sh "mv ${WORKSPACE}/${mdg}_*.snap ${WORKSPACE}/${mdg}.snap"
 110                 sh "sudo rm -rf ${WORKSPACE}/stage/ ${WORKSPACE}/parts/ ${WORKSPACE}/prime/"
 111
 112                 REV=""
 113                 if ( ! isMergeJob ) {
 114                     REV="/"+"${GERRIT_REFSPEC}".replaceAll('/','-')
 115                 }
 116                 channel="latest"
 117                 if (BRANCH_NAME.startsWith("v")) {
 118                     channel=BRANCH_NAME.substring(1)
 119                 } else if (BRANCH_NAME!="master") {
 120                     REV="/"+BRANCH_NAME+REV.replaceAll('/','-')
 121                 }
 122
 123                 sh "snapcraft push --release=${channel}/edge${REV} ${mdg}.snap"
 124                 sh "sudo rm -rf ${WORKSPACE}/*.snap"
 125             }
 126         }
 127     }
 128
 129     stage('Charm build') {
 130         charms = [
 131             'installers/charm/bundles/osm',
 132             'installers/charm/bundles/osm-ha',
 133             'installers/charm/grafana',
 134             'installers/charm/mongodb-exporter',
 135             'installers/charm/mysqld-exporter',
 136             'installers/charm/osm-lcm',
 137             'installers/charm/osm-mon',
 138             'installers/charm/osm-nbi',
 139             'installers/charm/osm-ng-ui',
 140             'installers/charm/osm-pol',
 141             'installers/charm/osm-ro',
 142             'installers/charm/osm-update-db-operator',
 143             'installers/charm/prometheus',
 144             'installers/charm/vca-integrator-operator',
 145         ]
 146         parallelSteps = [:]
 147         concurrency = sh(returnStdout: true, script: "expr `nproc` / 2").trim() as int
 148         for (charmPath in charms) {
 149             def directory = charmPath
 150             if (fileExists("$charmPath/charmcraft.yaml")) {
 151                 if (isMergeJob) {
 152                     CHECK_CHANGES = "git diff --name-only HEAD^1 -- $directory |wc -l"
 153                 } else {
 154                     CHECK_CHANGES = "git diff --name-only origin/$GERRIT_BRANCH -- $directory |wc -l"
 155                 }
 156                 charmsChanged = sh(returnStdout:true,  script: CHECK_CHANGES).trim()
 157
 158                 if (charmsChanged != "0") {
 159                     println("$directory has changes, rebuilding")
 160
 161                     parallelSteps[directory] = {
 162                         waitUntil(quiet: true, initialRecurrencePeriod: 5) {
 163                             if (concurrency > 0) {
 164                                 concurrency--
 165                                 true
 166                             } else {
 167                                 sleep 5
 168                                 false
 169                             }
 170                         }
 171                         try {
 172                             dir("${directory}") {
 173                                 try {
 174                                     sh "charmcraft pack --format json --verbosity brief 1> pack.json 2> pack-stderr.log"
 175                                 } catch (Exception e) {
 176                                     sh "cat pack-stderr.log"
 177                                     throw e
 178                                 }
 179
 180                                 def charmFilename = sh(
 181                                     returnStdout: true,
 182                                     script: 'cat pack.json | jq -r ".charms[0], .bundles[0]" | grep -v null').trim()
 183
 184                                 withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) {
 185                                     try {
 186                                         println("Uploading $charmFilename")
 187                                         sh "charmcraft upload --format json --verbosity brief $charmFilename 1> upload.json 2> upload-stderr.log"
 188                                     } catch (Exception e) {
 189                                         def errorMessage = sh(
 190                                             returnStdout: true,
 191                                             script: 'cat upload.json | jq -r ".errors[0].message" | grep -v null').trim()
 192                                         if (errorMessage.contains("already exists")) {
 193                                             println("No changes, skipping upload")
 194                                             return 0
 195                                         }
 196                                         sh "cat upload-stderr.log"
 197                                         throw e
 198                                     }
 199                                     def charmRevision = sh(
 200                                         returnStdout: true,
 201                                         script: 'cat upload.json | jq -r ".revision"').trim()
 202
 203                                     def charmBranch="merged"
 204                                     if ( !JOB_NAME.contains('merge') ) {
 205                                         charmBranch="${GERRIT_REFSPEC}".replaceAll('/','-')
 206                                     }
 207                                     def channel="latest"
 208                                     if (BRANCH_NAME.startsWith("v")) {
 209                                         channel=BRANCH_NAME.substring(1)
 210                                     } else if (BRANCH_NAME!="master" && JOB_NAME.contains('merge')) {
 211                                         charmBranch=BRANCH_NAME
 212                                     }
 213
 214                                     def charmName = sh(
 215                                         returnStdout:true,
 216                                         script: "if [ -f bundle.yaml ];then cat bundle.yaml;else cat metadata.yaml;fi|yq -r .name").trim()
 217
 218                                     def resourceArgument = ""
 219                                     try {
 220                                         def resourceName = sh(
 221                                             returnStdout:true,
 222                                             script: "cat metadata.yaml | yq '.resources | keys | .[0]'"
 223                                         ).trim()
 224                                         if( resourceName != "null" ) {
 225                                             sh "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'"
 226                                             def resourceVers = sh(
 227                                                 returnStdout:true,
 228                                                 script: "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'"
 229                                             ).trim()
 230
 231                                             resourceArgument = "--resource $resourceName:$resourceVers"
 232
 233                                             resourceName = sh(
 234                                                 returnStdout:true,
 235                                                 script: "cat metadata.yaml | yq '.resources | keys | .[1]'").trim()
 236
 237                                             if( resourceName != "null" ) {
 238                                                 resourceVers = sh(
 239                                                     returnStdout:true,
 240                                                     script: "charmcraft resource-revisions $charmName $resourceName | head -2 | tail -1 | awk '{print \$1}'"
 241                                                 ).trim()
 242                                                 resourceArgument += " --resource $resourceName:$resourceVers"
 243                                             }
 244                                         }
 245
 246                                     } catch (Exception discard) {
 247                                         print("No resource associated with bundle, skipping")
 248                                     }
 249
 250                                     print("resourceArgument = $resourceArgument")
 251
 252                                     sh "charmcraft release ${charmName} --revision=${charmRevision} ${resourceArgument} --channel=${channel}/edge/${charmBranch}"
 253                                 }
 254                             }
 255                         } finally {
 256                             concurrency++
 257                         }
 258                     }
 259                 }
 260             }
 261         }
 262         parallel parallelSteps
 263     }
 264     if ( do_stage_3 ) {
 265         stage('Build System') {
 266             def downstream_params_stage_3 = [
 267                 string(name: 'GERRIT_BRANCH', value: "${branch}"),
 268                 string(name: 'INSTALLER', value: "Default" ),
 269                 string(name: 'OPENSTACK_BASE_IMAGE', value: "ubuntu22.04" ),
 270                 string(name: 'OPENSTACK_OSM_FLAVOR', value: "osm.sanity" ),
 271                 string(name: 'UPSTREAM_JOB_NAME', value: "${JOB_NAME}" ),
 272                 string(name: 'UPSTREAM_JOB_NUMBER', value: "${BUILD_NUMBER}" ),
 273                 booleanParam(name: 'DO_STAGE_4', value: do_stage_4 )
 274             ]
 275             stage_3_job = "osm-stage_3"
 276             if ( JOB_NAME.contains('merge') ) {
 277                 stage_3_job += '-merge'
 278             }
 279
 280             // callout to stage_3. This is the system build
 281             result = build job: "${stage_3_job}/${branch}", parameters: downstream_params_stage_3, propagate: true
 282             if (result.getResult() != 'SUCCESS') {
 283                 project = result.getProjectName()
 284                 build = result.getNumber()
 285                 error("${project} build ${build} failed")
 286             }
 287         }
 288     }
 289 }
 290
 291 return this