allow repository configuration 35/1935/1
authorMike Marchetti <mmarchetti@sandvine.com>
Thu, 15 Jun 2017 17:02:16 +0000 (13:02 -0400)
committerMike Marchetti <mmarchetti@sandvine.com>
Thu, 15 Jun 2017 17:03:36 +0000 (13:03 -0400)
- for system test, allow using local repository
Change-Id: I7d8882ac501c58853fb9b4a1494b54708d1d2aec
Signed-off-by: Mike Marchetti <mmarchetti@sandvine.com>
installers/install_osm.sh
jenkins/RO/install
jenkins/SO/install
jenkins/UI/install
jenkins/common/install_common [new file with mode: 0644]
jenkins/host/install
jenkins/system/Jenkinsfile
jenkins/system/start_build

index 8f48272..f39eb41 100755 (executable)
@@ -19,6 +19,10 @@ function usage(){
     echo -e "  OPTIONS"
     echo -e "     --uninstall:   uninstall OSM: remove the containers and delete NAT rules"
     echo -e "     --source:      install OSM from source code using the latest stable tag"
+    echo -e "     -r <repo>:      use specified repository name for osm packages"
+    echo -e "     -R <release>:   use specified release for osm packages"
+    echo -e "     -u <repo base>: use specified repository url for osm packages"
+    echo -e "     -k <repo key>:  use specified repository public key url"
     echo -e "     -b <refspec>:  install OSM from source code using a specific branch (master, v2.0, ...) or tag"
     echo -e "                    -b master          (main dev branch)"
     echo -e "                    -b v2.0            (v2.0 branch)"
@@ -250,7 +254,7 @@ COMMIT_ID=""
 ASSUME_YES=""
 INSTALL_FROM_SOURCE=""
 
-while getopts ":hy-:b:" o; do
+while getopts ":hy-:b:r:k:u:R:" o; do
     case "${o}" in
         h)
             usage && exit 0
@@ -258,6 +262,18 @@ while getopts ":hy-:b:" o; do
         b)
             COMMIT_ID=${OPTARG}
             ;;
+        r)
+            REPOSITORY="-r ${OPTARG}"
+            ;;
+        R)
+            RELEASE="-R ${OPTARG}"
+            ;;
+        k)
+            REPOSITORY_KEY="-k ${OPTARG}"
+            ;;
+        u)
+            REPOSITORY_BASE="-u ${OPTARG}"
+            ;;
         -)
             [ "${OPTARG}" == "help" ] && usage && exit 0
             [ "${OPTARG}" == "source" ] && INSTALL_FROM_SOURCE="y" && continue
@@ -355,10 +371,10 @@ wget -q -O- https://osm-download.etsi.org/ftp/osm-2.0-two/README.txt &> /dev/nul
 
 if [ -z "$INSTALL_FROM_SOURCE" ]; then
     echo -e "\nCreating the containers and installing from binaries ..."
-    $OSM_DEVOPS/jenkins/host/install RO || FATAL "RO install failed"
+    $OSM_DEVOPS/jenkins/host/install RO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "RO install failed"
     $OSM_DEVOPS/jenkins/host/start_build VCA || FATAL "VCA install failed"
-    $OSM_DEVOPS/jenkins/host/install SO || FATAL "SO install failed"
-    $OSM_DEVOPS/jenkins/host/install UI || FATAL "UI install failed"
+    $OSM_DEVOPS/jenkins/host/install SO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "SO install failed"
+    $OSM_DEVOPS/jenkins/host/install UI $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "UI install failed"
 else #install from source
     echo -e "\nCreating the containers and building from source ..."
     $OSM_DEVOPS/jenkins/host/start_build RO --notest checkout $COMMIT_ID || FATAL "RO container build failed (refspec: '$COMMIT_ID')"
index 5bbc11f..d2eaf58 100755 (executable)
@@ -26,12 +26,11 @@ OSM_JENKINS=$(dirname $HERE)
 export OSM_MDG=RO
 OSM_load_config
 
+. $OSM_JENKINS/common/install_common
+
 # Installation starts here
 
 #Release TWO
-curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add -
-sleep 5
-add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable RO openvim"
 
 apt-get update
 apt-get -y install python-lib-osm-openvim
index 1f44210..d881a42 100755 (executable)
 # install launchpad SO and all of its dependencies
 #
 
+HERE=$(realpath $(dirname $0))
+OSM_JENKINS=$(dirname $HERE)
+. $OSM_JENKINS/common/install_common
+
 # Defensive bash programming flags
 set -o errexit    # Exit on any error
 trap 'echo ERROR: Command failed: \"$BASH_COMMAND\"' ERR
 set -o nounset    # Expanding an unset variable is an error.  Variables must be
                   # set before they can be used.
 
-if [ $# -gt 0 ]; then
-    case "$1" in
-       -h|--help)
-           echo
-           echo "NAME:"
-           echo "  $0"
-           echo
-           echo "SYNOPSIS:"
-           echo "  $0 -h|--help"
-           echo "  $0 [PLATFORM_REPOSITORY] [PLATFORM_VERSION]"
-           echo
-           echo "DESCRIPTION:"
-           echo "  Prepare current system to run SO and UI."
-           echo
-           echo "  PLATFORM_REPOSITORY (optional): name of the RIFT.ware repository."
-           echo "  PLATFORM_VERSION (optional): version of the platform packages to be installed."
-           echo
-           exit 0;;
-    esac
-fi
-
-
 ###############################################################################
 # Set up repo and version
+PLATFORM_REPOSITORY=OSM
+PLATFORM_VERSION=4.4.2.1.61839
+
+while getopts ":P:V:h" o; do
+    case "${o}" in
+        P)
+            PLATFORM_REPOSITORY=${OPTARG}
+            ;;
+        V)
+            PLATFORM_VERSION=${OPTARG}
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+function usage() {
+    echo
+    echo "NAME:"
+    echo "  $0"
+    echo
+    echo "SYNOPSIS:"
+    echo "  $0 -h|--help"
+    echo "  $0 -P <PLATFORM_REPOSITORY> -V <PLATFORM_VERSION>"
+    echo
+    echo "DESCRIPTION:"
+    echo "  Prepare current system to run SO and UI."
+    echo
+    echo "  PLATFORM_REPOSITORY (optional): name of the RIFT.ware repository."
+    echo "  PLATFORM_VERSION (optional): version of the platform packages to be installed."
+    echo
+}
 
-PLATFORM_REPOSITORY=${1:-OSM}
-PLATFORM_VERSION=${2:-4.4.2.1.61839}
 
 ###############################################################################
 # Main block
@@ -79,14 +97,12 @@ apt-get install -y --allow-downgrades rw.tools-container-tools=${PLATFORM_VERSIO
 /usr/rift/container_tools/mkcontainer --modes ext --modes platform --rw-version ${PLATFORM_VERSION}
 pip3 install lxml==3.4.0
 
-
 chmod 777 /usr/rift /usr/rift/usr/share
 
 # now disable the RIFT OSM repo so that we'll get MANO from
 # the OSM repos
 rm -f /etc/apt/sources.list.d/RIFT.list
-curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add -
-add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable SO UI RO openvim"
+
 apt-get update
 
 apt-get install -y \
index 5928784..4ba7123 100755 (executable)
 # install launchpad UI and create the service
 #
 
-curl https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO/OSM%20ETSI%20Release%20Key.gpg |apt-key add -
-add-apt-repository -y "deb https://osm-download.etsi.org/repository/osm/debian/ReleaseTWO stable SO UI RO openvim"
+HERE=$(realpath $(dirname $0))
+OSM_JENKINS=$(dirname $HERE)
+. $OSM_JENKINS/common/install_common
+
 rm -f /etc/apt/sources.list.d/RIFT.list 
 apt-get update
 
diff --git a/jenkins/common/install_common b/jenkins/common/install_common
new file mode 100644 (file)
index 0000000..ef8322b
--- /dev/null
@@ -0,0 +1,50 @@
+# this file is meant to be sourced
+#
+#   Copyright 2017 Sandvine
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+
+RELEASE="ReleaseTWO"
+REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg"
+REPOSITORY="stable"
+REPOSITORY_BASE="http://osm-download.etsi.org/repository/osm/debian"
+
+while getopts ":r:k:u:R:" o; do
+    case "${o}" in
+        r)
+            REPOSITORY=${OPTARG}
+            ;;
+        R)
+            RELEASE=${OPTARG}
+            ;;
+        k)
+            REPOSITORY_KEY=${OPTARG}
+            ;;
+        u)
+            REPOSITORY_BASE=${OPTARG}
+            ;;
+        *)
+            exit 1
+            ;;
+    esac
+done
+
+key_location=$REPOSITORY_BASE/$RELEASE/$REPOSITORY_KEY
+
+# key pulled from https
+[[ "$key_location" =~ https ]] || key_location=${key_location/http/https}
+curl $key_location | apt-key add -
+
+add-apt-repository -y "deb $REPOSITORY_BASE/$RELEASE $REPOSITORY SO UI RO osmclient openvim"
index ade30fb..dfdbe59 100755 (executable)
@@ -32,13 +32,13 @@ if ! container_exists $OSM_BUILD_CONTAINER; then
        [[ "$OSM_BUILD_CONTAINER_ALLOW_NESTED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.nesting=true"
        create_container $OSM_BASE_IMAGE $OSM_BUILD_CONTAINER $CONTAINER_OPTS
        wait_container_up $OSM_BUILD_CONTAINER
-       if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then
+       if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then
                container_push_devops $OSM_BUILD_CONTAINER
        else
                container_exec $OSM_BUILD_CONTAINER git clone ${OSM_GIT_URL}/devops
        fi
 else
-       if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then
+       if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then
                container_push_devops $OSM_BUILD_CONTAINER
        else
                container_exec $OSM_BUILD_CONTAINER git -C devops pull
index d3688c4..c3dcc0a 100644 (file)
@@ -1,7 +1,13 @@
 // input parameters:
 //   boolean: BUILD_FROM_SOURCE
+//   boolean: REPO_DISTRO
 //   boolean: COMMIT_ID
+//   boolean: UPSTREAM_SUFFIX
 //   string:  NODE
+//   string: RSYNC_DESTINATION
+//   string: REPO_BASE_URL
+//   string: REPO_KEY_NAME
+//   string: RELEASE
 
 node("${params.NODE}") {
 
@@ -11,6 +17,42 @@ node("${params.NODE}") {
         container_name = "${container_name_prefix}-${BUILD_NUMBER}"
     }
 
+    stage("Checkout") {
+        checkout scm
+    }
+
+    // Copy the artifacts from the upstream jobs
+    stage("Copy Artifacts") {
+        // cleanup any previous repo
+        sh 'rm -rf repo'
+
+        dir('repo') {
+            // grab all stable upstream builds based on the
+            // given target UPSTREAM_SUFFIX
+
+            def list = ["SO", "UI", "RO", "openvim", "osmclient"]
+            for (component in list) {
+                step ([$class: 'CopyArtifact',
+                       projectName: "${component}_${params.UPSTREAM_SUFFIX}"])
+                sh "dpkg-sig --sign builder -k dpkg1 pool/${component}/*"
+                // cleanup any prevously defined dists
+                sh "rm -rf dists"
+            }
+
+            // now create the distro
+            for (component in list) {
+                sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/"
+                sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages"
+                sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages"
+            }
+
+            // create and sign the release file
+            sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release"
+            sh "gpg --yes -abs -u dpkg1 -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release"
+            sh "rsync -avz . ${params.RSYNC_DESTINATION}/${params.RELEASE}"
+        }
+    }
+
     stage("Cleanup") {
         // check for previous containers and clean them up
         sh "jenkins/system/delete_old_containers.sh ${container_name_prefix}"
@@ -22,9 +64,22 @@ node("${params.NODE}") {
         {
             from_source = '--source'
         }
+        sh """
+            export OSM_USE_LOCAL_DEVOPS=true
+            jenkins/host/start_build system --build-container ${container_name} \
+                                            -b ${params.COMMIT_ID} \
+                                            -r ${params.REPO_DISTRO} \
+                                            -u ${params.REPO_BASE_URL} \
+                                            -k ${params.REPO_KEY_NAME} \
+                                            -R ${params.RELEASE} \
+                                            ${from_source}
+           """
+    }
 
-        sh "jenkins/host/start_build system --build-container ${container_name} -b ${params.COMMIT_ID} ${from_source}"
+    stage("Archive Artifacts") {
         sh "echo ${container_name} > build_version.txt"
-        archiveArtifacts artifacts: "build_version.txt"
+        sh "tar -zcvf repo.tar.gz repo"
+        archiveArtifacts artifacts: "build_version.txt, repo.tar.gz"
     }
 }
index a50a5a8..7f259c7 100755 (executable)
@@ -57,7 +57,8 @@ apt-get install -y python-pip python python-pycurl charm-tools python-pytest
 git clone https://osm.etsi.org/gerrit/osm/osmclient
 pip install osmclient/.
 
-devops/installers/install_osm.sh $*
+export OSM_USE_LOCAL_DEVOPS=true
+devops/installers/install_osm.sh --test $*
 RC=$?
 
 # workaround.  for upload packages, lxdbr0 needs to be promiscuous