From 85dc239b834cfe437455d0bb815c667ca905edc6 Mon Sep 17 00:00:00 2001 From: beierlm Date: Wed, 14 Apr 2021 13:51:53 -0400 Subject: [PATCH] Local Build Script Script to help perform builds locally, so that developers can recreate the Jenkins experience at home Change-Id: Ib1a3a555d28e6a5233f6fc3ee9062bb124c946d0 Signed-off-by: beierlm --- tools/local-build.sh | 421 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100755 tools/local-build.sh diff --git a/tools/local-build.sh b/tools/local-build.sh new file mode 100755 index 00000000..d1e8238c --- /dev/null +++ b/tools/local-build.sh @@ -0,0 +1,421 @@ +#!/bin/bash +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +HTTPDDIR="$( cd "${HOME}/snap/qhttp/common" &> /dev/null && pwd )" +HTTPPORT=11480 +KUBECFG="~/.osm/microk8s-config.yaml" +NO_CACHE="" +OPENSTACKRC="/var/snap/microstack/common/etc/microstack.rc" +REGISTRY="localhost:32000" +ROOTDIR="$( cd "${DIR}/../../" &> /dev/null && pwd)" + +function check_arguments(){ + while [ $# -gt 0 ] ; do + case $1 in + --debug) set -x ;; + --help | -h) show_help && exit 0 ;; + --httpddir) HTTPDIR="$2" && shift;; + --install-local-registry) 'install_local_registry' ;; + --install-microstack) 'install_microstack' ;; + --install-qhttpd) INSTALL_HTTPD='install_qhttpd' ;; + --kubecfg) KUBECFG="$2" && shift ;; + --module) TARGET_MODULE="$2" && shift;; + --no-cache) NO_CACHE="--no-cache" ;; + --openstackrc) OPENSTACKRC="$2" && shift ;; + --registry) REGISTRY="$2" && shift;; + --robot-local-mounts) ROBOT_LOCAL=YES ;; + --run-tests) TESTS=YES ;; + stage-2) STAGE_2='stage_2 ${TARGET_MODULE}' ;; + stage-3) STAGE_3='stage_3 ${TARGET_MODULE}' ;; + registry-push) REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;; + install-osm) INSTALL_OSM='install_osm' ;; + start-robot) START_ROBOT='start_robot' ;; + update-install) UPDATE_INSTALL='update_osm_module ${TARGET_MODULE}' + REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;; + *) echo "Unknown option $1" + show_help + exit 1;; + esac + shift + done +} + +function show_help() { + cat << EOF +Usage: $0 [OPTIONS] +Perform a local build and potential installation of OSM from sources, using the +same process as Jenkins. + +OPTIONS: + --help display this help message + --debug enable set -x for this script + --install-local-regitstry install and enable Microk8s local registry on port 32000 + --install-microstack install Microstack and configure to run robot tests + --install-qhttpd install QHTTPD as an HTTP server on port ${HTTPPORT} + --kubecfg path to kubecfg.yaml (uses Charmed OSM by default) + --no-cache do not use any cache when building docker images + --module only build this comma delimited list of modules + --openstackrc path to Openstack RC file (uses Microstack by default) + --registry use this alternate docker registry + --run-tests run stage 2 tests + stage-2 run the stage 2 build + stage-3 run the stage 3 build + registry-push push to the local registry + install-osm perform full installation of Charmed OSM from registry + start-robot start the Robot test container and leave you at prompt + update-install update Charmed OSM with new module container +EOF +} + +function install_local_registry() { + sudo snap install microk8s --classic + microk8s status --wait-ready + microk8s.enable registry +} + +function install_microstack() { + sudo snap install microstack --devmode --edge + sudo snap set microstack config.network.ports.dashboard=10080 + sudo microstack.init --auto --control + + sudo snap alias microstack.openstack openstack + . /var/snap/microstack/common/etc/microstack.rc + + for i in $(openstack security group list | awk '/default/{ print $2 }'); do + openstack security group rule create $i --protocol icmp --remote-ip 0.0.0.0/0 + openstack security group rule create $i --protocol tcp --remote-ip 0.0.0.0/0 + done + + openstack network create --enable --no-share osm-ext + openstack subnet create osm-ext-subnet --network osm-ext --dns-nameserver 8.8.8.8 \ + --subnet-range 172.30.0.0/24 + openstack router create external-router + openstack router add subnet external-router osm-ext-subnet + openstack router set --external-gateway external external-router + + curl -L https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img \ + | openstack image create --public --container-format=bare \ + --disk-format=qcow2 cirros-0.3.5-x86_64-disk.img + curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \ + | openstack image create --public --container-format=bare \ + --disk-format=qcow2 ubuntu16.04 + curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \ + | openstack image create --public --container-format=bare \ + --disk-format=qcow2 US1604 + curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img \ + | openstack image create --public --container-format=bare \ + --disk-format=qcow2 ubuntu18.04 + curl https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img \ + | openstack image create --public --container-format=bare \ + --disk-format=qcow2 ubuntu20.04 +} + +function install_qhttpd() { + sudo snap install qhttp + EXISTING_PID=$(ps auxw | grep "http.server 11480" | grep -v grep | awk '{print $2}') + if [ ! -z $EXISTING_PID ] ; then + kill $EXISTING_PID + fi + qhttp -p ${HTTPPORT} & +} + +function stage_2() { + cat << EOF +======================================================================================= +Performing Stage 2 +======================================================================================= +EOF + MODULES="common devops IM LCM MON N2VC NBI NG-UI osmclient PLA POL RO tests" + if [ ! -z ${1} ] ; then + POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g") + for MODULE in ${POSSIBLE_MODULES}; do + if ! echo "${MODULES}" | grep -q "${MODULE}" ; then + echo "Unknown stage 2 module ${MODULE}" + echo "Must be one of ${MODULES}" + exit 1 + fi + done + MODULES=${POSSIBLE_MODULES} + else + cat << EOF +======================================================================================= +Cleaning HTTP Directory for full build +======================================================================================= +EOF + rm -fv ${HTTPDDIR}/*.deb + fi + + for MODULE in ${MODULES} ; do + cd "${ROOTDIR}" + if [ ! -d ${MODULE} ] ; then + echo "Directory ${ROOTDIR}/${MODULE} does not exist" + exit 1 + fi + cat << EOF +======================================================================================= +Building ${MODULE} +======================================================================================= +EOF + cd ${MODULE} + find . -name '*.deb' -exec rm -v {} \; + docker build ${NO_CACHE} -t ${MODULE,,}-stage2 . + + STAGES="stage-build.sh" + if [ ! -z $TESTS ] ; then + STAGES="stage-test.sh ${STAGES}" + fi + for STAGE in $STAGES ; do + docker run -ti \ + -v "$(pwd):/build" \ + -w /build \ + ${MODULE,,}-stage2 \ + bash -c "groupadd -o -g $(id -g) -r $USER ; + useradd -o -u $(id -u) -d /build -r -g $USER $USER ; + runuser $USER -c devops-stages/${STAGE}" + if [ $? -ne 0 ] ; then + cat << EOF +======================================================================================= +Failed to build ${MODULE} +======================================================================================= +EOF + exit 1 + fi + done + + find . -name '*.deb' -exec mv -v {} ${HTTPDDIR}/ \; + done +} + +function _find_module_dockerfile() { + cd "${ROOTDIR}/devops/docker" + MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '` + if [ ! -z ${1} ] ; then + POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g") + for MODULE in ${POSSIBLE_MODULES}; do + if ! echo "${MODULES}" | grep -q "${MODULE}" ; then + echo "Unknown stage 3 module ${MODULE}" + echo "Must be one of ${MODULES}" + exit 1 + fi + done + echo ${POSSIBLE_MODULES} + else + echo ${MODULES} + fi +} + +function stage_3() { + cat << EOF +======================================================================================= +Performing Stage 3 +======================================================================================= +EOF + MODULES=$(_find_module_dockerfile $1) + BUILD_ARGS="" + HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+') + for file in ~/snap/qhttp/common/*.deb ; do + file=`basename ${file}` + name=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g"`; + name=${name^^}_URL + BUILD_ARGS="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file " + echo Added ${name} as http://$HOSTIP:${HTTPPORT}/$file + done + + for MODULE in ${MODULES} ; do + cd "${ROOTDIR}/devops/docker" + if [ ! -d ${MODULE} ] ; then + echo "Directory ${ROOTDIR}/${MODULE} does not exist" + exit 1 + fi + cat << EOF +======================================================================================= +Building ${MODULE} +======================================================================================= +EOF + cd ${MODULE} + MODULE=${MODULE,,} + docker build ${NO_CACHE} -t opensourcemano/${MODULE}:devel ${BUILD_ARGS} . + if [ $? -ne 0 ] ; then + cat << EOF +======================================================================================= +Failed to build ${MODULE} +======================================================================================= +EOF + exit 1 + fi + done +} + +function local_registry_push() { + cat << EOF +======================================================================================= +Pushing to local registry +======================================================================================= +EOF + cd "${ROOTDIR}/devops/docker" + MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '` + if [ ! -z ${1} ] ; then + POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g") + for MODULE in ${POSSIBLE_MODULES}; do + echo "${MODULE}" + if ! echo "${MODULES}" | grep -q "${MODULE}" ; then + echo "Unknown stage 3 module ${MODULE}" + echo "Must be one of ${MODULES}" + exit 1 + fi + done + MODULES=${POSSIBLE_MODULES} + fi + for MODULE in ${MODULES} ; do + MODULE=${MODULE,,} + docker tag opensourcemano/${MODULE}:devel ${REGISTRY}/opensourcemano/${MODULE}:devel + docker push ${REGISTRY}/opensourcemano/${MODULE}:devel + done +} + +function install_osm() { + cd "${ROOTDIR}/devops/installers" + VCA="" + if juju controllers 2>/dev/null| grep osm-vca ; then + VCA="--vca osm-vca" + fi + ./charmed_install.sh --registry localhost:32000 --tag devel ${VCA} +} + +function start_robot() { + mkdir -p "${ROOTDIR}/tests/local" + cd "${ROOTDIR}/tests/local" + + . ${OPENSTACKRC} + + # Workaround for microstack auth URL + if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then + export OS_AUTH_URL=${OS_AUTH_URL}/v3 + fi + + export OSM_HOSTNAME=$(juju config -m osm nbi site_url | sed "s/http.*\?:\/\///"):443 + export PROMETHEUS_HOSTNAME=$(juju config -m osm prometheus site_url | sed "s/http.*\?:\/\///") + export PROMETHEUS_PORT=80 + export JUJU_PASSWORD=`juju gui 2>&1 | grep password | awk '{print $2}'` + export HOSTIP=$(echo $PROMETHEUS_HOSTNAME | sed "s/prometheus.//" | sed "s/.xip.io//") + + rm robot-systest.cfg + for line in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg ; done + cat << EOF >> robot-systest.cfg +VIM_TARGET=osm +VIM_MGMT_NET=osm-ext +ENVIRONMENTS_FOLDER=environments +PACKAGES_FOLDER=/robot-systest/osm-packages +OS_CLOUD=openstack +LC_ALL=C.UTF-8 +LANG=C.UTF-8 +EOF + + cat << EOF > robot.etc.hosts +127.0.0.1 localhost +${HOSTIP} prometheus.${HOSTIP}.xip.io nbi.${HOSTIP}.xip.io +EOF + cat << EOF > clouds.yaml +clouds: + openstack: + auth: + auth_url: $OS_AUTH_URL + project_name: $OS_PROJECT_NAME + username: $OS_USERNAME + password: $OS_PASSWORD + user_domain_name: $OS_USER_DOMAIN_NAME + project_domain_name: $OS_PROJECT_DOMAIN_NAME +EOF + + VIM_AUTH_URL=$(osm vim-show osm | grep vim_url | awk '{print $4}' | tr -d \") + if [ ! -z ${VIM_AUTH_URL} -a "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ] ; then + echo "Deleting existing VIM osm as auth URLs have changed" + osm vim-delete osm + fi + + if ! osm vim-show osm &> /dev/null ; then + echo "Creating VIM osm" + osm vim-create --name osm --user "$OS_USERNAME" --password "$OS_PASSWORD" \ + --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \ + --config='{use_floating_ip: True, + management_network_name: osm-ext}' + fi + + if [ ! -z $ROBOT_LOCAL ] ; then + LOCAL_MOUNT_1="/robot-systest/lib" + LOCAL_MOUNT_2="/robot-systest/resources" + LOCAL_MOUNT_3="/robot-systest/testsuite" + else + LOCAL_MOUNT_1="/tmp/lib" + LOCAL_MOUNT_2="/tmp/resources" + LOCAL_MOUNT_3="/tmp/testsuite" + fi + + mkdir -p reports + + docker run -ti --entrypoint /bin/bash \ + --env OSM_HOSTNAME=${OSM_HOSTNAME} \ + --env PROMETHEUS_HOSTNAME=${PROMETHEUS_HOSTNAME} \ + --env PROMETHEUS_PORT=${PROMETHEUS_PORT} \ + --env JUJU_PASSWORD=${JUJU_PASSWORD} \ + --env HOSTIP=${HOSTIP} \ + --env-file robot-systest.cfg \ + -v "$(pwd)/robot.etc.hosts":/etc/hosts \ + -v ~/.osm/microk8s-config.yaml:/root/.kube/config \ + -v "$(pwd)/clouds.yaml":/etc/openstack/clouds.yaml \ + -v "$(pwd)/reports:"/robot-systest/reports \ + -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \ + -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \ + -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \ + opensourcemano/tests:devel +} + +function update_osm_module() { + MODULES=$(_find_module_dockerfile $1) + for MODULE in ${MODULES} ; do + MODULE=${MODULE,,} + echo "Updating ${MODULE}" + juju attach-resource ${MODULE} image=localhost:32000/opensourcemano/${MODULE}:devel + done +} + +if [ "$0" != "$BASH_SOURCE" ]; then + + _osm_local_build() + { + OPTIONS=$(show_help | sed '0,/^OPTIONS:$/d' | awk '{print $1}') + COMPREPLY=($(compgen -W "${OPTIONS}" -- "${COMP_WORDS[-1]}")) + } + + THIS_SCRIPT="$(basename ${BASH_SOURCE[0]})" + echo "Setting up bash completion for ${THIS_SCRIPT}" + complete -F _osm_local_build "${THIS_SCRIPT}" +else + check_arguments $@ + + eval "${INSTALL_HTTPD}" + eval "${INSTALL_LOCAL_REGISTRY}" + eval "${INSTALL_MICROSTACK}" + eval "${STAGE_2}" + eval "${STAGE_3}" + eval "${REGISTRY_PUSH}" + eval "${INSTALL_OSM}" + eval "${UPDATE_INSTALL}" + eval "${START_ROBOT}" +fi -- 2.17.1