From 50c2e86ae38c7715bf60cf8f70bff7b319bfa683 Mon Sep 17 00:00:00 2001 From: Jeremy Mordkoff Date: Fri, 24 Jun 2016 15:26:41 -0400 Subject: [PATCH] initial skeleton for the OSM DevOps build system Signed-off-by: Jeremy Mordkoff --- jenkins/README | 7 ++++ jenkins/SETTINGS | 21 ++++++++++++ jenkins/common/all_funcs | 30 ++++++++++++++++ jenkins/common/config | 40 ++++++++++++++++++++++ jenkins/common/container | 66 ++++++++++++++++++++++++++++++++++++ jenkins/common/logging | 53 +++++++++++++++++++++++++++++ jenkins/host/start_build | 45 ++++++++++++++++++++++++ jenkins/riftware/SETTINGS | 22 ++++++++++++ jenkins/riftware/start_build | 42 +++++++++++++++++++++++ jenkins/template/SETTINGS | 28 +++++++++++++++ 10 files changed, 354 insertions(+) create mode 100644 jenkins/README create mode 100644 jenkins/SETTINGS create mode 100644 jenkins/common/all_funcs create mode 100644 jenkins/common/config create mode 100644 jenkins/common/container create mode 100644 jenkins/common/logging create mode 100755 jenkins/host/start_build create mode 100644 jenkins/riftware/SETTINGS create mode 100755 jenkins/riftware/start_build create mode 100644 jenkins/template/SETTINGS diff --git a/jenkins/README b/jenkins/README new file mode 100644 index 00000000..482f817e --- /dev/null +++ b/jenkins/README @@ -0,0 +1,7 @@ +This directory holds the scripts and tools needed by jenkins to run jobs + +The 'host' subdirectory is meant to be invoked directly by jenkins either in its own container or on the host. + +The 'template' subdirectory contains sample files needed to configure a new MDG + +The other subdirectories correspond to MDGs and contain the scripts that are run inside the container diff --git a/jenkins/SETTINGS b/jenkins/SETTINGS new file mode 100644 index 00000000..50bef014 --- /dev/null +++ b/jenkins/SETTINGS @@ -0,0 +1,21 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Settings that are global to this installation +# MDG specific settings can be found in each MDG directory +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# base url for all repositories +OSM_GIT_URL=https://osm.etsi.org/gerrit/osm diff --git a/jenkins/common/all_funcs b/jenkins/common/all_funcs new file mode 100644 index 00000000..10b7650b --- /dev/null +++ b/jenkins/common/all_funcs @@ -0,0 +1,30 @@ +# this file is meant to be sourced +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# import all functions +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +if [ -z "$OSM_JENKINS" ]; then + export OSM_JENKINS=$(realpath $(dirname ${BASH_SOURCE[0]} )) +fi + +. ${OSM_JENKINS}/common/logging +INFO "logging sourced" +. ${OSM_JENKINS}/common/config +INFO "config sourced" +. ${OSM_JENKINS}/common/container +INFO "container sourced" diff --git a/jenkins/common/config b/jenkins/common/config new file mode 100644 index 00000000..7f49381e --- /dev/null +++ b/jenkins/common/config @@ -0,0 +1,40 @@ +#!/bin/bash +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# config functions +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + + +OSM_load_config() { + OSM_load_config_file ${OSM_JENKINS}/SETTINGS + if [ -z "$OSM_MDG" ]; then + WARNING "OSM_MDG not set" + else + OSM_load_config_file ${OSM_JENKINS}/${OSM_MDG}/SETTINGS + fi +} + +OSM_load_config_file() { + [ $# -eq 1 ] || FATAL "arg is filename" + if [ -f "$1" ]; then + . "$1" + INFO "config file $1 loaded" + else + WARNING "$1 not found" + fi +} + diff --git a/jenkins/common/container b/jenkins/common/container new file mode 100644 index 00000000..89db8bab --- /dev/null +++ b/jenkins/common/container @@ -0,0 +1,66 @@ +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# container_funcs +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +container_exists() { + if [ $# -ne 1 ]; then + FATAL "arg is container name" + fi + lxc config show $1 2>&1 >/dev/null + if [ $? -eq 0 ]; then + DEBUG "container $1 exists" + return 0 + else + DEBUG "container $1 not found" + return 1 + fi +} + +create_container() { + if [ $# -ne 2 ]; then + FATAL "args are image container" + fi + INFO "creating container $2 using image $1" + DEBUG "lxc launch $1 $2" + sudo lxc launch "$1" "$2" +} + +container_exec() { + container="$1" + shift + DEBUG "exec in $1 \"$*\"" + lxc exec "$container" -- $* +} + +wait_container_up() { + [ $# -eq 1 ] || FATAL "arg is container name got $# args - $*" + ct=0 + RE="1 received" + while [ $ct -lt 60 ]; do + let ct=ct+1 + output=$(container_exec "$1" ping -c 1 google.com) + if [[ $output =~ $RE ]]; then + DEBUG "$1 is up" + return + fi + INFO "waiting for container $1 to start" + DEBUG "expected '$RE' in $output" + sleep 1 + done + FATAL "container $1 did not start" +} diff --git a/jenkins/common/logging b/jenkins/common/logging new file mode 100644 index 00000000..3eb4a0d2 --- /dev/null +++ b/jenkins/common/logging @@ -0,0 +1,53 @@ +#!/bin/bash +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# container_funcs +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +print_stack() { + local i + local stack_size=${#FUNCNAME[1]} + echo "BACKTRACE:" >&2 + for (( i=1; i<$stack_size ; i++ )); do + local func="${FUNCNAME[$i]}" + [ x$func = x ] && func=MAIN + local linen="${BASH_LINENO[(( i - 1 ))]}" + local src="${BASH_SOURCE[$i]}" + [ x"$src" = x ] && src=non_file_source + echo "### $func $src $linen" >&2 + done + echo "-------" >&2 +} + + +FATAL() { + echo "### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2 + print_stack + exit 1 +} + +WARNING() { + echo "### $(date) ${FUNCNAME[1]}: WARNING error: $*" >&2 +} + +INFO() { + echo "## $(date) ${FUNCNAME[1]}: $*" >&2 +} + +DEBUG() { + echo "# $(date) ${FUNCNAME[1]}: $*" >&2 +} diff --git a/jenkins/host/start_build b/jenkins/host/start_build new file mode 100755 index 00000000..09f471bb --- /dev/null +++ b/jenkins/host/start_build @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# start-build is run on a host to start a MDG build +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +[ $# -eq 1 ] || FATAL "arg is MDG name" + +export OSM_MDG=$1 +OSM_load_config + +if ! container_exists $OSM_BUILD_CONTAINER; then + create_container $OSM_BASE_IMAGE $OSM_BUILD_CONTAINER + wait_container_up $OSM_BUILD_CONTAINER + RE="fedora|fc[0-9]" + if [[ $OSM_BASE_IMAGE =~ $RE ]]; then + container_exec $OSM_BUILD_CONTAINER yum -y install git + else + container_exec $OSM_BUILD_CONTAINER apt install git + fi + container_exec $OSM_BUILD_CONTAINER git clone ${OSM_GIT_URL}/devops +else + container_exec $OSM_BUILD_CONTAINER git -C devops pull +fi + + +container_exec $OSM_BUILD_CONTAINER ./devops/jenkins/$OSM_MDG/start_build diff --git a/jenkins/riftware/SETTINGS b/jenkins/riftware/SETTINGS new file mode 100644 index 00000000..d1af4176 --- /dev/null +++ b/jenkins/riftware/SETTINGS @@ -0,0 +1,22 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Settings specific to the riftware MDG +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# see ../template/SETTINGS for descriptions +export OSM_BASE_IMAGE=fc20 +export OSM_BUILD_CONTAINER=rift-build +export OSM_RUNTIME_CONTAINER=rift-runtime diff --git a/jenkins/riftware/start_build b/jenkins/riftware/start_build new file mode 100755 index 00000000..80a4a1f0 --- /dev/null +++ b/jenkins/riftware/start_build @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# script to start a build. This is run inside a container +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs +export OSM_MDG=riftware +OSM_load_config + + +if [ -d riftware ]; then + INFO "reusing existing workspace" + cd riftware + git pull +else + INFO "cloning MDG riftware from $OSM_GIT_URL/riftware" + git clone $OSM_GIT_URL/riftware + cd riftware + INFO "install pre-requisites" + bash ./scripts/vm_image/mkvmimg --container --modes build +fi + +INFO "done" + + + diff --git a/jenkins/template/SETTINGS b/jenkins/template/SETTINGS new file mode 100644 index 00000000..09cae6dd --- /dev/null +++ b/jenkins/template/SETTINGS @@ -0,0 +1,28 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=fc20 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=rift-build +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=rift-runtime -- 2.25.1