From f8140d1bb20f610480044734a87ab7247590db64 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 22 Jan 2021 16:04:13 -0500 Subject: [PATCH] Snap promotion fix Changes snapcraft to run directly on host instead of in docker continer. This adds the requirement to have snapcraft installed in the Jenkins agent, but is more reliable as the snapcraft docker image is not the best practice anymore for building snaps. Back ported new license scan. Change-Id: I21da8a2d38671fc7b253cb6e19a8afb1f1e6acb6 Signed-off-by: beierlm --- jenkins/ci-pipelines/ci_stage_3.groovy | 35 +++++++++----------- tools/license_scan.sh | 46 ++++++-------------------- 2 files changed, 27 insertions(+), 54 deletions(-) diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 71e53868..ff329b92 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -114,9 +114,6 @@ def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus def archive_logs(remote) { --def archive_logs(stackName) { -- sh "docker service ls |grep \"${stackName}\"| awk '{print \$2}'| xargs -iy docker service logs y --timestamps > containers_logs.txt 2>&1" -- archiveArtifacts artifacts: 'containers_logs.txt' sshCommand remote: remote, command: '''mkdir -p logs''' if (useCharmedInstaller) { sshCommand remote: remote, command: ''' @@ -128,9 +125,15 @@ def archive_logs(remote) { ''' } else { sshCommand remote: remote, command: ''' - for service in `docker service ls| grep \"${stackName}\"| awk '{print \$2}`; do - echo "Extracting log for $service" - docker service logs $service --timestamps 2>&1 > logs/$service.log + for deployment in `kubectl -n osm get deployments | grep -v operator | grep -v NAME| awk '{print $1}'`; do + echo "Extracting log for $deployment" + kubectl -n osm logs deployments/$deployment --timestamps=true --all-containers 2>&1 > logs/$deployment.log + done + ''' + sshCommand remote: remote, command: ''' + for statefulset in `kubectl -n osm get statefulsets | grep -v operator | grep -v NAME| awk '{print $1}'`; do + echo "Extracting log for $statefulset" + kubectl -n osm logs statefulsets/$statefulset --timestamps=true --all-containers 2>&1 > logs/$statefulset.log done ''' } @@ -515,8 +518,9 @@ EOF""" stage("Snap promotion") { def snaps = ["osmclient"] + sh "snapcraft login --with ~/.snapcraft/config" for (snap in snaps) { - channel="" + channel="latest/" if (BRANCH_NAME.startsWith("v")) { channel=BRANCH_NAME.substring(1)+"/" } else if (BRANCH_NAME!="master") { @@ -524,26 +528,19 @@ EOF""" } track=channel+"edge\\*" edge_rev=sh(returnStdout: true, - script: "sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " + - "-w /build snapcore/snapcraft:stable /bin/bash -c " + - "\"snapcraft login --with /snapcraft/config &>/dev/null && " + - "snapcraft revisions $snap\" | " + + script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + print "edge rev is $edge_rev" track=channel+"beta\\*" beta_rev=sh(returnStdout: true, - script: "sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " + - "-w /build snapcore/snapcraft:stable /bin/bash -c " + - "\"snapcraft login --with /snapcraft/config &>/dev/null && " + - "snapcraft revisions $snap\" | " + + script: "snapcraft revisions $snap | " + "grep \"$track\" | tail -1 | awk '{print \$1}'").trim() + print "beta rev is $beta_rev" if ( edge_rev != beta_rev ) { print "Promoting $edge_rev to beta in place of $beta_rev" beta_track=channel+"beta" - sh("sudo docker run -v ~/.snapcraft:/snapcraft -v ${WORKSPACE}:/build " + - "-w /build snapcore/snapcraft:stable /bin/bash -c " + - "\"snapcraft login --with /snapcraft/config &>/dev/null && " + - "snapcraft release $snap $edge_rev $beta_track\"") + sh "snapcraft release $snap $edge_rev $beta_track" } } } diff --git a/tools/license_scan.sh b/tools/license_scan.sh index 5af31dff..6e317b14 100755 --- a/tools/license_scan.sh +++ b/tools/license_scan.sh @@ -1,3 +1,4 @@ +#!/bin/sh # # Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. # @@ -13,55 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#!/bin/sh - -# Change log: -# 1. Bug 722 : Jayant Madavi : JM00553988@techmahindra.com : Enhancement to use new fossology server. Had to change the variable name at # couple of places, while scanning the variable name was adding curl as a license. -# 2. Bug 542 : Jayant Madavi, Mrityunjay Yadav : JM00553988@techmahindra.com : 24-jul-2019 : Enhancement to raise exit in case files modified or added does # not contain license. -# 3. Bug 542 : Jayant Madavi, Mrityunjay Yadav : JM00553988@techmahindra.com :add exception list. for now as inLine. later need to create a variable for exception_list - -echo GERRIT BRANCH is $GERRIT_BRANCH -dpkg -l wget &>/dev/null ||sudo apt-get install -y wget -dpkg -l curl &>/dev/null ||sudo apt-get install -y curl -#Curl can be used instead of wget: -#curl -s -X POST -d @$file https://osm.etsi.org/fossology/?mod=agent_nomos_once apache=0 nolicense=0 other=0 -exception_list="':(exclude)*.pdf' ':(exclude)*.png' ':(exclude)*.jpeg' ':(exclude)*.jpg' ':(exclude)*.gif' ':(exclude)*.json' ':(exclude)*.ico'" +exception_list="':(exclude)*.pdf' ':(exclude)*.png' ':(exclude)*.jpeg' ':(exclude)*.jpg' ':(exclude)*.gif' ':(exclude)*.json' ':(exclude)*.ico' ':(exclude)*.svg' ':(exclude)*.tiff'" git fetch -RE="FATAL: your file did not get passed through" - for file in $(echo ${exception_list} | xargs git diff --name-only origin/$GERRIT_BRANCH -- . ); do + license="No Apache license found" if [ -f $file ]; then if [ -s $file ]; then - licnse=$(curl -s -X POST -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Content-Type: multipart/form-data' -H 'cache-control: no-cache' -F "file_input=@\"$file\"" -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*\(.*\).*:\1:p' |xargs) - result=$(echo $licnse | grep "$RE") - if [ -n "$result" ]; then - # possibly we have exceeded the post rate - sleep 10 - licnse=$(curl -s -X POST -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Content-Type: multipart/form-data' -H 'cache-control: no-cache' -F "file_input=@\"$file\"" -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*\(.*\).*:\1:p' |xargs) + if [ $(grep -c "http://www.apache.org/licenses/LICENSE-2.0" $file) -ge 1 ] ; then + license="Apache-2.0" fi - else - licnse="No_license_found" fi else - licnse="DELETED" + license="DELETED" fi - echo "$file $licnse" - case "$licnse" in + echo "$file $license" + case "$license" in "Apache-2.0") apache=$((apache + 1)) ;; - "No_license_found") + No*) nolicense=$((nolicense + 1)) ;; "DELETED") ;; - "FATAL:*") - ;; *) echo "BAD LICENSE ON FILE $file" other=$((other + 1)) @@ -69,14 +49,10 @@ for file in $(echo ${exception_list} | xargs git diff --name-only origin/$GERRIT esac done -if [ $other -gt 0 ]; then - echo "FATAL: Non-apache licenses detected" - exit 2 -fi if [ $nolicense -gt 0 ]; then - echo "FATAL: Unlicensed files found" - exit 2 + echo "FATAL: Files without apache license found" + exit 2 fi exit 0 -- 2.25.1