Local Build Script

Script to help perform builds locally, so that developers can recreate
the Jenkins experience at home

Change-Id: Ib1a3a555d28e6a5233f6fc3ee9062bb124c946d0
Signed-off-by: beierlm <mark.beierl@canonical.com>

tools/local-build.sh

+#!/bin/bash
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+HTTPDDIR="$( cd "${HOME}/snap/qhttp/common" &> /dev/null && pwd )"
+HTTPPORT=11480
+KUBECFG="~/.osm/microk8s-config.yaml"
+NO_CACHE=""
+OPENSTACKRC="/var/snap/microstack/common/etc/microstack.rc"
+REGISTRY="localhost:32000"
+ROOTDIR="$( cd "${DIR}/../../" &> /dev/null && pwd)"
+
+function check_arguments(){
+    while [ $# -gt 0 ] ; do
+        case $1 in
+            --debug) set -x ;;
+            --help | -h) show_help && exit 0 ;;
+            --httpddir) HTTPDIR="$2" && shift;;
+            --install-local-registry) 'install_local_registry' ;;
+            --install-microstack) 'install_microstack' ;;
+            --install-qhttpd) INSTALL_HTTPD='install_qhttpd' ;;
+            --kubecfg) KUBECFG="$2" && shift ;;
+            --module) TARGET_MODULE="$2" && shift;;
+            --no-cache) NO_CACHE="--no-cache" ;;
+            --openstackrc) OPENSTACKRC="$2" && shift ;;
+            --registry) REGISTRY="$2" && shift;;
+            --robot-local-mounts) ROBOT_LOCAL=YES ;;
+            --run-tests) TESTS=YES ;;
+            stage-2) STAGE_2='stage_2 ${TARGET_MODULE}' ;;
+            stage-3) STAGE_3='stage_3 ${TARGET_MODULE}' ;;
+            registry-push) REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
+            install-osm) INSTALL_OSM='install_osm' ;;
+            start-robot) START_ROBOT='start_robot' ;;
+            update-install) UPDATE_INSTALL='update_osm_module ${TARGET_MODULE}'
+                 REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
+            *)  echo "Unknown option $1"
+                show_help
+                exit 1;;
+        esac
+        shift
+    done
+}
+
+function show_help() {
+    cat << EOF
+Usage: $0 [OPTIONS]
+Perform a local build and potential installation of OSM from sources, using the
+same process as Jenkins.
+
+OPTIONS:
+  --help                        display this help message
+  --debug                       enable set -x for this script
+  --install-local-regitstry     install and enable Microk8s local registry on port 32000
+  --install-microstack          install Microstack and configure to run robot tests
+  --install-qhttpd              install QHTTPD as an HTTP server on port ${HTTPPORT}
+  --kubecfg                     path to kubecfg.yaml (uses Charmed OSM by default)
+  --no-cache                    do not use any cache when building docker images
+  --module                      only build this comma delimited list of modules
+  --openstackrc                 path to Openstack RC file (uses Microstack by default)
+  --registry                    use this alternate docker registry
+  --run-tests                   run stage 2 tests
+  stage-2                       run the stage 2 build
+  stage-3                       run the stage 3 build
+  registry-push                 push to the local registry
+  install-osm                   perform full installation of Charmed OSM from registry
+  start-robot                   start the Robot test container and leave you at prompt
+  update-install                update Charmed OSM with new module container
+EOF
+}
+
+function install_local_registry() {
+    sudo snap install microk8s --classic
+    microk8s status --wait-ready
+    microk8s.enable registry
+}
+
+function install_microstack() {
+    sudo snap install microstack --devmode --edge
+    sudo snap set microstack config.network.ports.dashboard=10080
+    sudo microstack.init --auto --control
+
+    sudo snap alias microstack.openstack openstack
+    . /var/snap/microstack/common/etc/microstack.rc
+
+    for i in $(openstack security group list | awk '/default/{ print $2 }'); do
+        openstack security group rule create $i --protocol icmp --remote-ip 0.0.0.0/0
+        openstack security group rule create $i --protocol tcp --remote-ip 0.0.0.0/0
+    done
+
+    openstack network create --enable --no-share osm-ext
+    openstack subnet create osm-ext-subnet --network osm-ext --dns-nameserver 8.8.8.8 \
+              --subnet-range 172.30.0.0/24
+    openstack router create external-router
+    openstack router add subnet external-router osm-ext-subnet
+    openstack router set --external-gateway external external-router
+
+    curl -L https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img \
+        | openstack image create --public --container-format=bare \
+         --disk-format=qcow2 cirros-0.3.5-x86_64-disk.img
+    curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
+        | openstack image create --public --container-format=bare \
+         --disk-format=qcow2 ubuntu16.04
+    curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
+        | openstack image create --public --container-format=bare \
+         --disk-format=qcow2 US1604
+    curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img \
+        | openstack image create --public --container-format=bare \
+         --disk-format=qcow2 ubuntu18.04
+    curl https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img \
+        | openstack image create --public --container-format=bare \
+         --disk-format=qcow2 ubuntu20.04
+}
+
+function install_qhttpd() {
+    sudo snap install qhttp
+    EXISTING_PID=$(ps auxw | grep "http.server 11480" | grep -v grep | awk '{print $2}')
+    if [ ! -z $EXISTING_PID ] ; then
+        kill $EXISTING_PID
+    fi
+    qhttp -p ${HTTPPORT} &
+}
+
+function stage_2() {
+    cat << EOF
+=======================================================================================
+Performing Stage 2
+=======================================================================================
+EOF
+    MODULES="common devops IM LCM MON N2VC NBI NG-UI osmclient PLA POL RO tests"
+    if [ ! -z ${1} ] ; then
+        POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
+        for MODULE in ${POSSIBLE_MODULES}; do
+            if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
+                echo "Unknown stage 2 module ${MODULE}"
+                echo "Must be one of ${MODULES}"
+                exit 1
+            fi
+        done
+        MODULES=${POSSIBLE_MODULES}
+    else
+        cat << EOF
+=======================================================================================
+Cleaning HTTP Directory for full build
+=======================================================================================
+EOF
+        rm -fv ${HTTPDDIR}/*.deb
+    fi
+
+    for MODULE in ${MODULES} ; do
+        cd "${ROOTDIR}"
+        if [ ! -d ${MODULE} ] ; then
+            echo "Directory ${ROOTDIR}/${MODULE} does not exist"
+            exit 1
+        fi
+        cat << EOF
+=======================================================================================
+Building ${MODULE}
+=======================================================================================
+EOF
+        cd ${MODULE}
+        find . -name '*.deb' -exec rm -v {} \;
+        docker build ${NO_CACHE} -t ${MODULE,,}-stage2 .
+
+        STAGES="stage-build.sh"
+        if [ ! -z $TESTS ] ; then
+            STAGES="stage-test.sh ${STAGES}"
+        fi
+        for STAGE in $STAGES ; do
+            docker run -ti \
+                -v "$(pwd):/build" \
+                -w /build \
+                ${MODULE,,}-stage2 \
+                bash -c "groupadd -o -g $(id -g) -r $USER ;
+                useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
+                runuser $USER -c devops-stages/${STAGE}"
+            if [ $? -ne 0 ] ; then
+            cat << EOF
+=======================================================================================
+Failed to build ${MODULE}
+=======================================================================================
+EOF
+                exit 1
+            fi
+        done
+
+        find . -name '*.deb' -exec mv -v {} ${HTTPDDIR}/ \;
+    done
+}
+
+function _find_module_dockerfile() {
+    cd "${ROOTDIR}/devops/docker"
+    MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
+    if [ ! -z ${1} ] ; then
+        POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
+        for MODULE in ${POSSIBLE_MODULES}; do
+            if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
+                echo "Unknown stage 3 module ${MODULE}"
+                echo "Must be one of ${MODULES}"
+                exit 1
+            fi
+        done
+        echo ${POSSIBLE_MODULES}
+    else
+        echo ${MODULES}
+    fi
+}
+
+function stage_3() {
+    cat << EOF
+=======================================================================================
+Performing Stage 3
+=======================================================================================
+EOF
+    MODULES=$(_find_module_dockerfile $1)
+    BUILD_ARGS=""
+    HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
+    for file in ~/snap/qhttp/common/*.deb ; do
+        file=`basename ${file}`
+        name=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g"`;
+        name=${name^^}_URL
+        BUILD_ARGS="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
+        echo Added ${name} as http://$HOSTIP:${HTTPPORT}/$file
+    done
+
+    for MODULE in ${MODULES} ; do
+        cd "${ROOTDIR}/devops/docker"
+        if [ ! -d ${MODULE} ] ; then
+            echo "Directory ${ROOTDIR}/${MODULE} does not exist"
+            exit 1
+        fi
+        cat << EOF
+=======================================================================================
+Building ${MODULE}
+=======================================================================================
+EOF
+        cd ${MODULE}
+        MODULE=${MODULE,,}
+        docker build ${NO_CACHE} -t opensourcemano/${MODULE}:devel ${BUILD_ARGS} .
+        if [ $? -ne 0 ] ; then
+        cat << EOF
+=======================================================================================
+Failed to build ${MODULE}
+=======================================================================================
+EOF
+            exit 1
+        fi
+    done
+}
+
+function local_registry_push() {
+    cat << EOF
+=======================================================================================
+Pushing to local registry
+=======================================================================================
+EOF
+    cd "${ROOTDIR}/devops/docker"
+    MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
+    if [ ! -z ${1} ] ; then
+        POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
+        for MODULE in ${POSSIBLE_MODULES}; do
+            echo "${MODULE}"
+            if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
+                echo "Unknown stage 3 module ${MODULE}"
+                echo "Must be one of ${MODULES}"
+                exit 1
+            fi
+        done
+        MODULES=${POSSIBLE_MODULES}
+    fi
+    for MODULE in ${MODULES} ; do
+        MODULE=${MODULE,,}
+        docker tag opensourcemano/${MODULE}:devel ${REGISTRY}/opensourcemano/${MODULE}:devel
+        docker push ${REGISTRY}/opensourcemano/${MODULE}:devel
+    done
+}
+
+function install_osm() {
+    cd "${ROOTDIR}/devops/installers"
+    VCA=""
+    if juju controllers 2>/dev/null| grep osm-vca ; then
+        VCA="--vca osm-vca"
+    fi
+    ./charmed_install.sh --registry localhost:32000  --tag devel ${VCA}
+}
+
+function start_robot() {
+    mkdir -p "${ROOTDIR}/tests/local"
+    cd "${ROOTDIR}/tests/local"
+
+    . ${OPENSTACKRC}
+
+    # Workaround for microstack auth URL
+    if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
+        export OS_AUTH_URL=${OS_AUTH_URL}/v3
+    fi
+
+    export OSM_HOSTNAME=$(juju config -m osm nbi site_url | sed "s/http.*\?:\/\///"):443
+    export PROMETHEUS_HOSTNAME=$(juju config -m osm prometheus site_url | sed "s/http.*\?:\/\///")
+    export PROMETHEUS_PORT=80
+    export JUJU_PASSWORD=`juju gui 2>&1 | grep password | awk '{print $2}'`
+    export HOSTIP=$(echo $PROMETHEUS_HOSTNAME | sed "s/prometheus.//" | sed "s/.xip.io//")
+
+    rm robot-systest.cfg
+    for line in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg ; done
+    cat << EOF >> robot-systest.cfg
+VIM_TARGET=osm
+VIM_MGMT_NET=osm-ext
+ENVIRONMENTS_FOLDER=environments
+PACKAGES_FOLDER=/robot-systest/osm-packages
+OS_CLOUD=openstack
+LC_ALL=C.UTF-8
+LANG=C.UTF-8
+EOF
+
+    cat << EOF > robot.etc.hosts
+127.0.0.1           localhost
+${HOSTIP}      prometheus.${HOSTIP}.xip.io nbi.${HOSTIP}.xip.io
+EOF
+    cat << EOF > clouds.yaml
+clouds:
+  openstack:
+    auth:
+      auth_url: $OS_AUTH_URL
+      project_name: $OS_PROJECT_NAME
+      username: $OS_USERNAME
+      password: $OS_PASSWORD
+      user_domain_name: $OS_USER_DOMAIN_NAME
+      project_domain_name: $OS_PROJECT_DOMAIN_NAME
+EOF
+
+    VIM_AUTH_URL=$(osm vim-show osm | grep vim_url | awk '{print $4}' | tr -d \")
+    if [ ! -z ${VIM_AUTH_URL} -a "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ] ; then
+        echo "Deleting existing VIM osm as auth URLs have changed"
+        osm vim-delete osm
+    fi
+
+    if ! osm vim-show osm &> /dev/null ; then
+        echo "Creating VIM osm"
+        osm vim-create --name osm --user "$OS_USERNAME" --password "$OS_PASSWORD" \
+        --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
+        --config='{use_floating_ip: True,
+                   management_network_name: osm-ext}'
+    fi
+
+    if [ ! -z $ROBOT_LOCAL ] ; then
+        LOCAL_MOUNT_1="/robot-systest/lib"
+        LOCAL_MOUNT_2="/robot-systest/resources"
+        LOCAL_MOUNT_3="/robot-systest/testsuite"
+    else
+        LOCAL_MOUNT_1="/tmp/lib"
+        LOCAL_MOUNT_2="/tmp/resources"
+        LOCAL_MOUNT_3="/tmp/testsuite"
+    fi
+
+    mkdir -p reports
+
+    docker run -ti --entrypoint /bin/bash \
+        --env OSM_HOSTNAME=${OSM_HOSTNAME} \
+        --env PROMETHEUS_HOSTNAME=${PROMETHEUS_HOSTNAME} \
+        --env PROMETHEUS_PORT=${PROMETHEUS_PORT} \
+        --env JUJU_PASSWORD=${JUJU_PASSWORD} \
+        --env HOSTIP=${HOSTIP} \
+        --env-file robot-systest.cfg \
+        -v "$(pwd)/robot.etc.hosts":/etc/hosts \
+        -v ~/.osm/microk8s-config.yaml:/root/.kube/config \
+        -v "$(pwd)/clouds.yaml":/etc/openstack/clouds.yaml \
+        -v "$(pwd)/reports:"/robot-systest/reports \
+        -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
+        -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
+        -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
+        opensourcemano/tests:devel
+}
+
+function update_osm_module() {
+    MODULES=$(_find_module_dockerfile $1)
+    for MODULE in ${MODULES} ; do
+        MODULE=${MODULE,,}
+        echo "Updating ${MODULE}"
+        juju attach-resource ${MODULE} image=localhost:32000/opensourcemano/${MODULE}:devel
+    done
+}
+
+if [ "$0" != "$BASH_SOURCE" ]; then
+
+    _osm_local_build()
+    {
+        OPTIONS=$(show_help | sed '0,/^OPTIONS:$/d' | awk '{print $1}')
+        COMPREPLY=($(compgen -W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
+    }
+
+    THIS_SCRIPT="$(basename ${BASH_SOURCE[0]})"
+    echo "Setting up bash completion for ${THIS_SCRIPT}"
+    complete -F _osm_local_build "${THIS_SCRIPT}"
+else
+    check_arguments $@
+
+    eval "${INSTALL_HTTPD}"
+    eval "${INSTALL_LOCAL_REGISTRY}"
+    eval "${INSTALL_MICROSTACK}"
+    eval "${STAGE_2}"
+    eval "${STAGE_3}"
+    eval "${REGISTRY_PUSH}"
+    eval "${INSTALL_OSM}"
+    eval "${UPDATE_INSTALL}"
+    eval "${START_ROBOT}"
+fi