From: Mike Marchetti Date: Thu, 15 Jun 2017 17:02:16 +0000 (-0400) Subject: allow repository configuration X-Git-Tag: v2.0.2~12 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=425f8cef392f81356280f6508384ac397aa2eb85;p=osm%2Fdevops.git allow repository configuration - for system test, allow using local repository Change-Id: I7d8882ac501c58853fb9b4a1494b54708d1d2aec Signed-off-by: Mike Marchetti --- diff --git a/installers/install_osm.sh b/installers/install_osm.sh index 8f482722..f39eb410 100755 --- a/installers/install_osm.sh +++ b/installers/install_osm.sh @@ -19,6 +19,10 @@ function usage(){ echo -e " OPTIONS" echo -e " --uninstall: uninstall OSM: remove the containers and delete NAT rules" echo -e " --source: install OSM from source code using the latest stable tag" + echo -e " -r : use specified repository name for osm packages" + echo -e " -R : use specified release for osm packages" + echo -e " -u : use specified repository url for osm packages" + echo -e " -k : use specified repository public key url" echo -e " -b : install OSM from source code using a specific branch (master, v2.0, ...) or tag" echo -e " -b master (main dev branch)" echo -e " -b v2.0 (v2.0 branch)" @@ -250,7 +254,7 @@ COMMIT_ID="" ASSUME_YES="" INSTALL_FROM_SOURCE="" -while getopts ":hy-:b:" o; do +while getopts ":hy-:b:r:k:u:R:" o; do case "${o}" in h) usage && exit 0 @@ -258,6 +262,18 @@ while getopts ":hy-:b:" o; do b) COMMIT_ID=${OPTARG} ;; + r) + REPOSITORY="-r ${OPTARG}" + ;; + R) + RELEASE="-R ${OPTARG}" + ;; + k) + REPOSITORY_KEY="-k ${OPTARG}" + ;; + u) + REPOSITORY_BASE="-u ${OPTARG}" + ;; -) [ "${OPTARG}" == "help" ] && usage && exit 0 [ "${OPTARG}" == "source" ] && INSTALL_FROM_SOURCE="y" && continue @@ -355,10 +371,10 @@ wget -q -O- https://osm-download.etsi.org/ftp/osm-2.0-two/README.txt &> /dev/nul if [ -z "$INSTALL_FROM_SOURCE" ]; then echo -e "\nCreating the containers and installing from binaries ..." - $OSM_DEVOPS/jenkins/host/install RO || FATAL "RO install failed" + $OSM_DEVOPS/jenkins/host/install RO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "RO install failed" $OSM_DEVOPS/jenkins/host/start_build VCA || FATAL "VCA install failed" - $OSM_DEVOPS/jenkins/host/install SO || FATAL "SO install failed" - $OSM_DEVOPS/jenkins/host/install UI || FATAL "UI install failed" + $OSM_DEVOPS/jenkins/host/install SO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "SO install failed" + $OSM_DEVOPS/jenkins/host/install UI $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "UI install failed" else #install from source echo -e "\nCreating the containers and building from source ..." $OSM_DEVOPS/jenkins/host/start_build RO --notest checkout $COMMIT_ID || FATAL "RO container build failed (refspec: '$COMMIT_ID')" diff --git a/jenkins/RO/install b/jenkins/RO/install index 5bbc11fe..d2eaf58f 100755 --- a/jenkins/RO/install +++ b/jenkins/RO/install @@ -26,12 +26,11 @@ OSM_JENKINS=$(dirname $HERE) export OSM_MDG=RO OSM_load_config +. $OSM_JENKINS/common/install_common + # Installation starts here #Release TWO -curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add - -sleep 5 -add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable RO openvim" apt-get update apt-get -y install python-lib-osm-openvim diff --git a/jenkins/SO/install b/jenkins/SO/install index 1f44210c..d881a42d 100755 --- a/jenkins/SO/install +++ b/jenkins/SO/install @@ -23,39 +23,57 @@ # install launchpad SO and all of its dependencies # +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/install_common + # Defensive bash programming flags set -o errexit # Exit on any error trap 'echo ERROR: Command failed: \"$BASH_COMMAND\"' ERR set -o nounset # Expanding an unset variable is an error. Variables must be # set before they can be used. -if [ $# -gt 0 ]; then - case "$1" in - -h|--help) - echo - echo "NAME:" - echo " $0" - echo - echo "SYNOPSIS:" - echo " $0 -h|--help" - echo " $0 [PLATFORM_REPOSITORY] [PLATFORM_VERSION]" - echo - echo "DESCRIPTION:" - echo " Prepare current system to run SO and UI." - echo - echo " PLATFORM_REPOSITORY (optional): name of the RIFT.ware repository." - echo " PLATFORM_VERSION (optional): version of the platform packages to be installed." - echo - exit 0;; - esac -fi - - ############################################################################### # Set up repo and version +PLATFORM_REPOSITORY=OSM +PLATFORM_VERSION=4.4.2.1.61839 + +while getopts ":P:V:h" o; do + case "${o}" in + P) + PLATFORM_REPOSITORY=${OPTARG} + ;; + V) + PLATFORM_VERSION=${OPTARG} + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +function usage() { + echo + echo "NAME:" + echo " $0" + echo + echo "SYNOPSIS:" + echo " $0 -h|--help" + echo " $0 -P -V " + echo + echo "DESCRIPTION:" + echo " Prepare current system to run SO and UI." + echo + echo " PLATFORM_REPOSITORY (optional): name of the RIFT.ware repository." + echo " PLATFORM_VERSION (optional): version of the platform packages to be installed." + echo +} -PLATFORM_REPOSITORY=${1:-OSM} -PLATFORM_VERSION=${2:-4.4.2.1.61839} ############################################################################### # Main block @@ -79,14 +97,12 @@ apt-get install -y --allow-downgrades rw.tools-container-tools=${PLATFORM_VERSIO /usr/rift/container_tools/mkcontainer --modes ext --modes platform --rw-version ${PLATFORM_VERSION} pip3 install lxml==3.4.0 - chmod 777 /usr/rift /usr/rift/usr/share # now disable the RIFT OSM repo so that we'll get MANO from # the OSM repos rm -f /etc/apt/sources.list.d/RIFT.list -curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add - -add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable SO UI RO openvim" + apt-get update apt-get install -y \ diff --git a/jenkins/UI/install b/jenkins/UI/install index 59287847..4ba71233 100755 --- a/jenkins/UI/install +++ b/jenkins/UI/install @@ -23,8 +23,10 @@ # install launchpad UI and create the service # -curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add - -add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable SO UI RO openvim" +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/install_common + rm -f /etc/apt/sources.list.d/RIFT.list apt-get update diff --git a/jenkins/common/install_common b/jenkins/common/install_common new file mode 100644 index 00000000..ef8322bb --- /dev/null +++ b/jenkins/common/install_common @@ -0,0 +1,50 @@ +# this file is meant to be sourced +# +# Copyright 2017 Sandvine +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +RELEASE="ReleaseTWO" +REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg" +REPOSITORY="stable" +REPOSITORY_BASE="http://osm-download.etsi.org/repository/osm/debian" + +while getopts ":r:k:u:R:" o; do + case "${o}" in + r) + REPOSITORY=${OPTARG} + ;; + R) + RELEASE=${OPTARG} + ;; + k) + REPOSITORY_KEY=${OPTARG} + ;; + u) + REPOSITORY_BASE=${OPTARG} + ;; + *) + exit 1 + ;; + esac +done + +key_location=$REPOSITORY_BASE/$RELEASE/$REPOSITORY_KEY + +# key pulled from https +[[ "$key_location" =~ https ]] || key_location=${key_location/http/https} +curl $key_location | apt-key add - + +add-apt-repository -y "deb $REPOSITORY_BASE/$RELEASE $REPOSITORY SO UI RO osmclient openvim" diff --git a/jenkins/host/install b/jenkins/host/install index ade30fb9..dfdbe596 100755 --- a/jenkins/host/install +++ b/jenkins/host/install @@ -32,13 +32,13 @@ if ! container_exists $OSM_BUILD_CONTAINER; then [[ "$OSM_BUILD_CONTAINER_ALLOW_NESTED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.nesting=true" create_container $OSM_BASE_IMAGE $OSM_BUILD_CONTAINER $CONTAINER_OPTS wait_container_up $OSM_BUILD_CONTAINER - if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then + if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then container_push_devops $OSM_BUILD_CONTAINER else container_exec $OSM_BUILD_CONTAINER git clone ${OSM_GIT_URL}/devops fi else - if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then + if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then container_push_devops $OSM_BUILD_CONTAINER else container_exec $OSM_BUILD_CONTAINER git -C devops pull diff --git a/jenkins/system/Jenkinsfile b/jenkins/system/Jenkinsfile index d3688c43..c3dcc0ac 100644 --- a/jenkins/system/Jenkinsfile +++ b/jenkins/system/Jenkinsfile @@ -1,7 +1,13 @@ // input parameters: // boolean: BUILD_FROM_SOURCE +// boolean: REPO_DISTRO // boolean: COMMIT_ID +// boolean: UPSTREAM_SUFFIX // string: NODE +// string: RSYNC_DESTINATION +// string: REPO_BASE_URL +// string: REPO_KEY_NAME +// string: RELEASE node("${params.NODE}") { @@ -11,6 +17,42 @@ node("${params.NODE}") { container_name = "${container_name_prefix}-${BUILD_NUMBER}" } + stage("Checkout") { + checkout scm + } + + // Copy the artifacts from the upstream jobs + stage("Copy Artifacts") { + // cleanup any previous repo + sh 'rm -rf repo' + + dir('repo') { + // grab all stable upstream builds based on the + // given target UPSTREAM_SUFFIX + + def list = ["SO", "UI", "RO", "openvim", "osmclient"] + for (component in list) { + step ([$class: 'CopyArtifact', + projectName: "${component}_${params.UPSTREAM_SUFFIX}"]) + sh "dpkg-sig --sign builder -k dpkg1 pool/${component}/*" + // cleanup any prevously defined dists + sh "rm -rf dists" + } + + // now create the distro + for (component in list) { + sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/" + sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + } + + // create and sign the release file + sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release" + sh "gpg --yes -abs -u dpkg1 -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release" + sh "rsync -avz . ${params.RSYNC_DESTINATION}/${params.RELEASE}" + } + } + stage("Cleanup") { // check for previous containers and clean them up sh "jenkins/system/delete_old_containers.sh ${container_name_prefix}" @@ -22,9 +64,22 @@ node("${params.NODE}") { { from_source = '--source' } + + sh """ + export OSM_USE_LOCAL_DEVOPS=true + jenkins/host/start_build system --build-container ${container_name} \ + -b ${params.COMMIT_ID} \ + -r ${params.REPO_DISTRO} \ + -u ${params.REPO_BASE_URL} \ + -k ${params.REPO_KEY_NAME} \ + -R ${params.RELEASE} \ + ${from_source} + """ + } - sh "jenkins/host/start_build system --build-container ${container_name} -b ${params.COMMIT_ID} ${from_source}" + stage("Archive Artifacts") { sh "echo ${container_name} > build_version.txt" - archiveArtifacts artifacts: "build_version.txt" + sh "tar -zcvf repo.tar.gz repo" + archiveArtifacts artifacts: "build_version.txt, repo.tar.gz" } } diff --git a/jenkins/system/start_build b/jenkins/system/start_build index a50a5a82..7f259c72 100755 --- a/jenkins/system/start_build +++ b/jenkins/system/start_build @@ -57,7 +57,8 @@ apt-get install -y python-pip python python-pycurl charm-tools python-pytest git clone https://osm.etsi.org/gerrit/osm/osmclient pip install osmclient/. -devops/installers/install_osm.sh $* +export OSM_USE_LOCAL_DEVOPS=true +devops/installers/install_osm.sh --test $* RC=$? # workaround. for upload packages, lxdbr0 needs to be promiscuous