From: marchettim Date: Wed, 20 Sep 2017 18:07:07 +0000 (+0200) Subject: Merge "Modified fixtures to make generalise VIM accessfor vim create" X-Git-Tag: v3.0.0rc~7 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=c121ff81ff1018d0f59af6d04503f3e8087e7c74;hp=f834e20a75108f81506e62e7139de95ad8e81002;p=osm%2Fdevops.git Merge "Modified fixtures to make generalise VIM accessfor vim create" --- diff --git a/jenkins/ci-pipelines/ci_helper.groovy b/jenkins/ci-pipelines/ci_helper.groovy index 974cd8d0..243a13d0 100644 --- a/jenkins/ci-pipelines/ci_helper.groovy +++ b/jenkins/ci-pipelines/ci_helper.groovy @@ -15,10 +15,8 @@ * under the License. */ -artifactory_server_id = 'artifactory-osm' - -def get_archive(mdg, branch, build_name, build_number, pattern='*') { - server = Artifactory.server artifactory_server_id +def get_archive(artifactory_server, mdg, branch, build_name, build_number, pattern='*') { + server = Artifactory.server artifactory_server println("retrieve archive for ${mdg}/${branch}/${build_name}/${build_number}/${pattern}") @@ -27,7 +25,7 @@ def get_archive(mdg, branch, build_name, build_number, pattern='*') { "files": [ { "target": "./", - "pattern": "${repo_prefix}${mdg}/${pattern}", + "pattern": "${repo_prefix}${mdg}/${branch}/${pattern}", "build": "${build_name}/${build_number}" } ] @@ -72,8 +70,8 @@ def get_ip_from_container( container_name ) { return sh(returnStdout: true, script: "lxc list ${container_name} -c 4|grep eth0 |awk '{print \$2}'").trim() } -def archive(mdg,branch,status) { - server = Artifactory.server artifactory_server_id +def archive(artifactory_server,mdg,branch,status) { + server = Artifactory.server artifactory_server def properties = "branch=${branch};status=${status}" def repo_prefix = 'osm-' @@ -81,19 +79,19 @@ def archive(mdg,branch,status) { "files": [ { "pattern": "dists/*.gz", - "target": "${repo_prefix}${mdg}/${BUILD_NUMBER}/", + "target": "${repo_prefix}${mdg}/${branch}/${BUILD_NUMBER}/", "props": "${properties}", "flat": false }, { "pattern": "dists/*Packages", - "target": "${repo_prefix}${mdg}/${BUILD_NUMBER}/", + "target": "${repo_prefix}${mdg}/${branch}/${BUILD_NUMBER}/", "props": "${properties}", "flat": false }, { "pattern": "pool/*/*.deb", - "target": "${repo_prefix}${mdg}/${BUILD_NUMBER}/", + "target": "${repo_prefix}${mdg}/${branch}/${BUILD_NUMBER}/", "props": "${properties}", "flat": false }] @@ -113,10 +111,10 @@ def archive(mdg,branch,status) { //CANNOT use build promotion with OSS version of artifactory // For now, will publish downloaded artifacts into a new repo. -def promote_build(mdg,branch,buildInfo) { +def promote_build(artifactory_server,mdg,branch,buildInfo) { println("Promoting build: mdg: ${mdg} branch: ${branch} build: ${buildInfo.name}/${buildInfo.number}") - server = Artifactory.server artifactory_server_id + server = Artifactory.server artifactory_server //def properties = "branch=${branch};status=${status}" def repo_prefix = 'osm-' diff --git a/jenkins/ci-pipelines/ci_stage_1.groovy b/jenkins/ci-pipelines/ci_stage_1.groovy index c345e940..12ebf16d 100644 --- a/jenkins/ci-pipelines/ci_stage_1.groovy +++ b/jenkins/ci-pipelines/ci_stage_1.groovy @@ -25,7 +25,8 @@ def Get_MDG(project) { return project } -node { +node("${params.NODE}") { + mdg = Get_MDG("${GERRIT_PROJECT}") println("MDG is ${mdg}") @@ -34,25 +35,34 @@ node { params.PROJECT_URL_PREFIX = 'https://osm.etsi.org/gerrit' } - // pipeline running from gerrit trigger. - // kickoff the downstream multibranch pipeline - def downstream_params = [ - string(name: 'GERRIT_BRANCH', value: GERRIT_BRANCH), - string(name: 'GERRIT_PROJECT', value: GERRIT_PROJECT), - string(name: 'GERRIT_REFSPEC', value: GERRIT_REFSPEC), - string(name: 'GERRIT_PATCHSET_REVISION', value: GERRIT_PATCHSET_REVISION), - string(name: 'PROJECT_URL_PREFIX', value: params.PROJECT_URL_PREFIX), - booleanParam(name: 'TEST_INSTALL', value: params.TEST_INSTALL), - ] + stage('downstream') { + // pipeline running from gerrit trigger. + // kickoff the downstream multibranch pipeline + def downstream_params = [ + string(name: 'GERRIT_BRANCH', value: GERRIT_BRANCH), + string(name: 'GERRIT_PROJECT', value: GERRIT_PROJECT), + string(name: 'GERRIT_REFSPEC', value: GERRIT_REFSPEC), + string(name: 'GERRIT_PATCHSET_REVISION', value: GERRIT_PATCHSET_REVISION), + string(name: 'PROJECT_URL_PREFIX', value: params.PROJECT_URL_PREFIX), + booleanParam(name: 'TEST_INSTALL', value: params.TEST_INSTALL), + ] - println("TEST_INSTALL = ${params.TEST_INSTALL}") - // callout to stage_2. This is a multi-branch pipeline. - upstream_job_name = "${mdg}-stage_2/${GERRIT_BRANCH}" + stage_name = "stage_2" + if ( params.STAGE ) + { + // go directly to stage 3 (osm system) + stage_name = "stage_3" + mdg = "osm" + } + println("TEST_INSTALL = ${params.TEST_INSTALL}") + // callout to stage_2. This is a multi-branch pipeline. + upstream_job_name = "${mdg}-${stage_name}/${GERRIT_BRANCH}" - stage_2_result = build job: "${upstream_job_name}", parameters: downstream_params, propagate: true - if (stage_2_result.getResult() != 'SUCCESS') { - project = stage_2_result.getProjectName() - build = stage_2_result.getNumber() - error("${project} build ${build} failed") + stage_2_result = build job: "${upstream_job_name}", parameters: downstream_params, propagate: true + if (stage_2_result.getResult() != 'SUCCESS') { + project = stage_2_result.getProjectName() + build = stage_2_result.getNumber() + error("${project} build ${build} failed") + } } } diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index c17a7310..5a5643d4 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -26,7 +26,7 @@ def project_checkout(url_prefix,project,refspec,revision) { } } -def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,build_system) { +def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,build_system,artifactory_server) { println("build_system = ${build_system}") ci_helper = load "devops/jenkins/ci-pipelines/ci_helper.groovy" @@ -38,19 +38,21 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,build_system) { project_checkout(url_prefix,project,refspec,revision) } + stage('License Scan') { + sh "devops/tools/license_scan.sh" + } + container_name = "${project}-${branch}".toLowerCase() stage('Docker-Build') { + sh ''' + echo RUN groupadd -o -g $(id -g) -r jenkins >> Dockerfile + echo RUN useradd -o -u $(id -u) --create-home -r -g jenkins jenkins >> Dockerfile + ''' sh "docker build -t ${container_name} ." } withDockerContainer("${container_name}") { - stage('Docker-Setup') { - sh ''' - groupadd -o -g $(id -g) -r jenkins - useradd -o -u $(id -u) --create-home -r -g jenkins jenkins - ''' - } stage('Test') { sh 'devops-stages/stage-test.sh' } @@ -61,7 +63,7 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,build_system) { stage('Archive') { sh(returnStdout:true, script: 'devops-stages/stage-archive.sh').trim() - ci_helper.archive(mdg,branch,'untested') + ci_helper.archive(artifactory_server,mdg,branch,'untested') } if ( build_system ) { diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 6b1198bd..00c3f306 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -29,6 +29,7 @@ properties([ string(defaultValue: '', description: '', name: 'UPSTREAM_JOB_NUMBER'), string(defaultValue: '', description: '', name: 'UPSTREAM_JOB_NUMBER'), string(defaultValue: 'dpkg1', description: '', name: 'GPG_KEY_NAME'), + string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_FAIL'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'), ]) @@ -59,7 +60,7 @@ node("${params.NODE}") { // given target UPSTREAM_SUFFIX dir("${RELEASE}") { - def list = ["SO", "UI", "RO", "openvim", "osmclient"] + def list = ["SO", "UI", "RO", "openvim", "osmclient", "IM"] for (component in list) { step ([$class: 'CopyArtifact', projectName: "${component}${params.UPSTREAM_SUFFIX}/${GERRIT_BRANCH}"]) @@ -68,7 +69,7 @@ node("${params.NODE}") { //options = get_env_from_build('build.env') build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER') //build_num = sh(returnStdout:true, script: "cat build.env | awk -F= '/BUILD_NUMBER/{print \$2}'").trim() - ci_helper.get_archive(component,GERRIT_BRANCH, "${component}-stage_2 :: ${GERRIT_BRANCH}", build_num) + ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${component}-stage_2 :: ${GERRIT_BRANCH}", build_num) // cleanup any prevously defined dists sh "rm -rf dists" @@ -89,7 +90,7 @@ node("${params.NODE}") { build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER') component = ci_helper.get_mdg_from_project(ci_helper.get_env_value('build.env','GERRIT_PROJECT')) - ci_helper.get_archive(component,GERRIT_BRANCH, "${component}-stage_2 :: ${GERRIT_BRANCH}", build_num) + ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${component}-stage_2 :: ${GERRIT_BRANCH}", build_num) sh "rm -rf dists" } @@ -179,7 +180,7 @@ node("${params.NODE}") { // Archive the tested repo dir("repo/${RELEASE}") { - ci_helper.archive(RELEASE,GERRIT_BRANCH,'tested') + ci_helper.archive(params.ARTIFACTORY_SERVER,RELEASE,GERRIT_BRANCH,'tested') } } } diff --git a/jenkins/common/install_common b/jenkins/common/install_common index d844bdd9..d9c1338c 100755 --- a/jenkins/common/install_common +++ b/jenkins/common/install_common @@ -45,4 +45,4 @@ key_location=$REPOSITORY_BASE/$RELEASE/$REPOSITORY_KEY curl $key_location | apt-key add - -apt update && add-apt-repository -y "deb $REPOSITORY_BASE/$RELEASE $REPOSITORY SO UI RO osmclient openvim" +apt-get update && add-apt-repository -y "deb $REPOSITORY_BASE/$RELEASE $REPOSITORY SO UI RO osmclient openvim" diff --git a/jenkins/host/start_build b/jenkins/host/start_build index 02bfcc26..f7ada68a 100755 --- a/jenkins/host/start_build +++ b/jenkins/host/start_build @@ -48,7 +48,7 @@ if ! container_exists $OSM_BUILD_CONTAINER; then if [[ $OSM_BASE_IMAGE =~ $RE ]]; then container_exec $OSM_BUILD_CONTAINER yum -y install git tar make sudo else - container_exec $OSM_BUILD_CONTAINER apt -y install git realpath make sudo + container_exec $OSM_BUILD_CONTAINER apt-get -y install git realpath make sudo fi if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then container_push_devops $OSM_BUILD_CONTAINER diff --git a/jenkins/osmclient/start_build b/jenkins/osmclient/start_build index 2206a1a7..155fefab 100755 --- a/jenkins/osmclient/start_build +++ b/jenkins/osmclient/start_build @@ -34,8 +34,8 @@ fi OSM_git_checkout "$@" -apt update -apt install -y docker.io +apt-get update +apt-get install -y docker.io # cleanup all pre-existing builds. # deb pkg build fails if previous build image is present diff --git a/jenkins/system/start_build b/jenkins/system/start_build index 823e160a..e1b91ac8 100755 --- a/jenkins/system/start_build +++ b/jenkins/system/start_build @@ -23,7 +23,7 @@ OSM_JENKINS=$(dirname $HERE) INFO "Installing packages" apt-get update -apt install -y lxd +apt-get install -y lxd INFO "Configuring LXD" # ZFS doesn't work inside a nested container. ZFS should be configured in the host LXD. @@ -55,9 +55,10 @@ systemctl start lxd-bridge apt-get install -y python-pip python python-pycurl charm-tools python-pytest -apt install -y python-osmclient +apt-get install -y python-osmclient export OSM_USE_LOCAL_DEVOPS=true +echo "devops/installers/install_osm.sh --test $*" devops/installers/install_osm.sh --test $* RC=$? diff --git a/tools/license_scan.sh b/tools/license_scan.sh new file mode 100755 index 00000000..47106e0e --- /dev/null +++ b/tools/license_scan.sh @@ -0,0 +1,76 @@ +# +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#!/bin/sh + +echo GERRIT BRANCH is $GERRIT_BRANCH +dpkg -l wget &>/dev/null ||sudo apt-get install -y wget +dpkg -l curl &>/dev/null ||sudo apt-get install -y curl +#Curl can be used instead of wget: +#curl -s -X POST -d @$file https://osm.etsi.org/fossology/?mod=agent_nomos_once + +apache=0 +nolicense=0 +other=0 + +git fetch + +RE="FATAL: your file did not get passed through" + +for file in $(git diff --name-only origin/$GERRIT_BRANCH); do + if [ -f $file ]; then + if [ -s $file ]; then + license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//") + result=$(echo $license | grep "$RE") + if [ -n "$result" ]; then + # possibly we have exceeded the post rate + sleep 10 + license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//") + fi + else + license="No_license_found" + fi + else + license="DELETED" + fi + echo "$file $license" + case "$license" in + "Apache-2.0") + apache=$((apache + 1)) + ;; + "No_license_found") + nolicense=$((nolicense + 1)) + ;; + "DELETED") + ;; + "FATAL:*") + ;; + *) + echo "BAD LICENSE ON FILE $file" + other=$((other + 1)) + ;; + esac +done + +if [ $other -gt 0 ]; then + echo "FATAL: Non-apache licenses detected" + exit 2 +fi + +if [ $nolicense -gt 0 ]; then + echo "WARNING: Unlicensed files found" +fi + +exit 0