tools/local-build.sh

   1 #!/bin/bash
   2 #######################################################################################
   3 # Copyright ETSI Contributors and Others.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #######################################################################################
  18
  19 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
  20 HTTPDDIR="$( cd "${HOME}/snap/qhttp/common" &> /dev/null && pwd )"
  21 HTTPPORT=11480
  22 KUBECFG="~/.osm/microk8s-config.yaml"
  23 NO_CACHE=""
  24 OPENSTACKRC="/var/snap/microstack/common/etc/microstack.rc"
  25 REGISTRY="localhost:32000"
  26 ROOTDIR="$( cd "${DIR}/../../" &> /dev/null && pwd)"
  27
  28 function check_arguments(){
  29     while [ $# -gt 0 ] ; do
  30         case $1 in
  31             --debug) set -x ;;
  32             --help | -h) show_help && exit 0 ;;
  33             --httpddir) HTTPDIR="$2" && shift;;
  34             --install-local-registry) 'install_local_registry' ;;
  35             --install-microstack) 'install_microstack' ;;
  36             --install-qhttpd) INSTALL_HTTPD='install_qhttpd' ;;
  37             --kubecfg) KUBECFG="$2" && shift ;;
  38             --module) TARGET_MODULE="$2" && shift;;
  39             --no-cache) NO_CACHE="--no-cache" ;;
  40             --openstackrc) OPENSTACKRC="$2" && shift ;;
  41             --registry) REGISTRY="$2" && shift;;
  42             --robot-local-mounts) ROBOT_LOCAL=YES ;;
  43             --run-tests) TESTS=YES ;;
  44             stage-2) STAGE_2='stage_2 ${TARGET_MODULE}' ;;
  45             stage-3) STAGE_3='stage_3 ${TARGET_MODULE}' ;;
  46             registry-push) REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  47             install-osm) INSTALL_OSM='install_osm' ;;
  48             start-robot) START_ROBOT='start_robot' ;;
  49             update-install) UPDATE_INSTALL='update_osm_module ${TARGET_MODULE}'
  50                  REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  51             *)  echo "Unknown option $1"
  52                 show_help
  53                 exit 1;;
  54         esac
  55         shift
  56     done
  57 }
  58
  59 function show_help() {
  60     cat << EOF
  61 Usage: $0 [OPTIONS]
  62 Perform a local build and potential installation of OSM from sources, using the
  63 same process as Jenkins.
  64
  65 OPTIONS:
  66   --help                        display this help message
  67   --debug                       enable set -x for this script
  68   --install-local-regitstry     install and enable Microk8s local registry on port 32000
  69   --install-microstack          install Microstack and configure to run robot tests
  70   --install-qhttpd              install QHTTPD as an HTTP server on port ${HTTPPORT}
  71   --kubecfg                     path to kubecfg.yaml (uses Charmed OSM by default)
  72   --no-cache                    do not use any cache when building docker images
  73   --module                      only build this comma delimited list of modules
  74   --openstackrc                 path to Openstack RC file (uses Microstack by default)
  75   --registry                    use this alternate docker registry
  76   --run-tests                   run stage 2 tests
  77   stage-2                       run the stage 2 build
  78   stage-3                       run the stage 3 build
  79   registry-push                 push to the local registry
  80   install-osm                   perform full installation of Charmed OSM from registry
  81   start-robot                   start the Robot test container and leave you at prompt
  82   update-install                update Charmed OSM with new module container
  83 EOF
  84 }
  85
  86 function install_local_registry() {
  87     sudo snap install microk8s --classic
  88     microk8s status --wait-ready
  89     microk8s.enable registry
  90 }
  91
  92 function install_microstack() {
  93     sudo snap install microstack --devmode --edge
  94     sudo snap set microstack config.network.ports.dashboard=10080
  95     sudo microstack.init --auto --control
  96
  97     sudo snap alias microstack.openstack openstack
  98     . /var/snap/microstack/common/etc/microstack.rc
  99
 100     for i in $(openstack security group list | awk '/default/{ print $2 }'); do
 101         openstack security group rule create $i --protocol icmp --remote-ip 0.0.0.0/0
 102         openstack security group rule create $i --protocol tcp --remote-ip 0.0.0.0/0
 103     done
 104
 105     openstack network create --enable --no-share osm-ext
 106     openstack subnet create osm-ext-subnet --network osm-ext --dns-nameserver 8.8.8.8 \
 107               --subnet-range 172.30.0.0/24
 108     openstack router create external-router
 109     openstack router add subnet external-router osm-ext-subnet
 110     openstack router set --external-gateway external external-router
 111
 112     curl -L https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img \
 113         | openstack image create --public --container-format=bare \
 114          --disk-format=qcow2 cirros-0.3.5-x86_64-disk.img
 115     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 116         | openstack image create --public --container-format=bare \
 117          --disk-format=qcow2 ubuntu16.04
 118     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 119         | openstack image create --public --container-format=bare \
 120          --disk-format=qcow2 US1604
 121     curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img \
 122         | openstack image create --public --container-format=bare \
 123          --disk-format=qcow2 ubuntu18.04
 124     curl https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img \
 125         | openstack image create --public --container-format=bare \
 126          --disk-format=qcow2 ubuntu20.04
 127 }
 128
 129 function install_qhttpd() {
 130     sudo snap install qhttp
 131     EXISTING_PID=$(ps auxw | grep "http.server 11480" | grep -v grep | awk '{print $2}')
 132     if [ ! -z $EXISTING_PID ] ; then
 133         kill $EXISTING_PID
 134     fi
 135     qhttp -p ${HTTPPORT} &
 136 }
 137
 138 function stage_2() {
 139     cat << EOF
 140 =======================================================================================
 141 Performing Stage 2
 142 =======================================================================================
 143 EOF
 144     MODULES="common devops IM LCM MON N2VC NBI NG-UI osmclient PLA POL RO tests"
 145     if [ ! -z ${1} ] ; then
 146         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 147         for MODULE in ${POSSIBLE_MODULES}; do
 148             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 149                 echo "Unknown stage 2 module ${MODULE}"
 150                 echo "Must be one of ${MODULES}"
 151                 exit 1
 152             fi
 153         done
 154         MODULES=${POSSIBLE_MODULES}
 155     else
 156         cat << EOF
 157 =======================================================================================
 158 Cleaning HTTP Directory for full build
 159 =======================================================================================
 160 EOF
 161         rm -fv ${HTTPDDIR}/*.deb
 162     fi
 163
 164     for MODULE in ${MODULES} ; do
 165         cd "${ROOTDIR}"
 166         if [ ! -d ${MODULE} ] ; then
 167             echo "Directory ${ROOTDIR}/${MODULE} does not exist"
 168             exit 1
 169         fi
 170         cat << EOF
 171 =======================================================================================
 172 Building ${MODULE}
 173 =======================================================================================
 174 EOF
 175         cd ${MODULE}
 176         find . -name '*.deb' -exec rm -v {} \;
 177         docker build ${NO_CACHE} -t ${MODULE,,}-stage2 .
 178
 179         STAGES="stage-build.sh"
 180         if [ ! -z $TESTS ] ; then
 181             STAGES="stage-test.sh ${STAGES}"
 182         fi
 183         for STAGE in $STAGES ; do
 184             docker run -ti \
 185                 -v "$(pwd):/build" \
 186                 -w /build \
 187                 ${MODULE,,}-stage2 \
 188                 bash -c "groupadd -o -g $(id -g) -r $USER ;
 189                 useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
 190                 runuser $USER -c devops-stages/${STAGE}"
 191             if [ $? -ne 0 ] ; then
 192             cat << EOF
 193 =======================================================================================
 194 Failed to build ${MODULE}
 195 =======================================================================================
 196 EOF
 197                 exit 1
 198             fi
 199         done
 200
 201         find . -name '*.deb' -exec mv -v {} ${HTTPDDIR}/ \;
 202     done
 203 }
 204
 205 function _find_module_dockerfile() {
 206     cd "${ROOTDIR}/devops/docker"
 207     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 208     if [ ! -z ${1} ] ; then
 209         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 210         for MODULE in ${POSSIBLE_MODULES}; do
 211             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 212                 echo "Unknown stage 3 module ${MODULE}"
 213                 echo "Must be one of ${MODULES}"
 214                 exit 1
 215             fi
 216         done
 217         echo ${POSSIBLE_MODULES}
 218     else
 219         echo ${MODULES}
 220     fi
 221 }
 222
 223 function stage_3() {
 224     cat << EOF
 225 =======================================================================================
 226 Performing Stage 3
 227 =======================================================================================
 228 EOF
 229     MODULES=$(_find_module_dockerfile $1)
 230     BUILD_ARGS=""
 231     HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
 232     for file in ~/snap/qhttp/common/*.deb ; do
 233         file=`basename ${file}`
 234         name=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g"`;
 235         name=${name^^}_URL
 236         BUILD_ARGS="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
 237         echo Added ${name} as http://$HOSTIP:${HTTPPORT}/$file
 238     done
 239
 240     for MODULE in ${MODULES} ; do
 241         cd "${ROOTDIR}/devops/docker"
 242         if [ ! -d ${MODULE} ] ; then
 243             echo "Directory ${ROOTDIR}/${MODULE} does not exist"
 244             exit 1
 245         fi
 246         cat << EOF
 247 =======================================================================================
 248 Building ${MODULE}
 249 =======================================================================================
 250 EOF
 251         cd ${MODULE}
 252         MODULE=${MODULE,,}
 253         docker build ${NO_CACHE} -t opensourcemano/${MODULE}:devel ${BUILD_ARGS} .
 254         if [ $? -ne 0 ] ; then
 255         cat << EOF
 256 =======================================================================================
 257 Failed to build ${MODULE}
 258 =======================================================================================
 259 EOF
 260             exit 1
 261         fi
 262     done
 263 }
 264
 265 function local_registry_push() {
 266     cat << EOF
 267 =======================================================================================
 268 Pushing to local registry
 269 =======================================================================================
 270 EOF
 271     cd "${ROOTDIR}/devops/docker"
 272     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 273     if [ ! -z ${1} ] ; then
 274         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 275         for MODULE in ${POSSIBLE_MODULES}; do
 276             echo "${MODULE}"
 277             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 278                 echo "Unknown stage 3 module ${MODULE}"
 279                 echo "Must be one of ${MODULES}"
 280                 exit 1
 281             fi
 282         done
 283         MODULES=${POSSIBLE_MODULES}
 284     fi
 285     for MODULE in ${MODULES} ; do
 286         MODULE=${MODULE,,}
 287         docker tag opensourcemano/${MODULE}:devel ${REGISTRY}/opensourcemano/${MODULE}:devel
 288         docker push ${REGISTRY}/opensourcemano/${MODULE}:devel
 289     done
 290 }
 291
 292 function install_osm() {
 293     cd "${ROOTDIR}/devops/installers"
 294     VCA=""
 295     if juju controllers 2>/dev/null| grep osm-vca ; then
 296         VCA="--vca osm-vca"
 297     fi
 298     ./charmed_install.sh --registry localhost:32000  --tag devel ${VCA}
 299 }
 300
 301 function start_robot() {
 302     mkdir -p "${ROOTDIR}/tests/local"
 303     cd "${ROOTDIR}/tests/local"
 304
 305     . ${OPENSTACKRC}
 306
 307     # Workaround for microstack auth URL
 308     if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
 309         export OS_AUTH_URL=${OS_AUTH_URL}/v3
 310     fi
 311
 312     export OSM_HOSTNAME=$(juju config -m osm nbi site_url | sed "s/http.*\?:\/\///"):443
 313     export PROMETHEUS_HOSTNAME=$(juju config -m osm prometheus site_url | sed "s/http.*\?:\/\///")
 314     export PROMETHEUS_PORT=80
 315     export JUJU_PASSWORD=`juju gui 2>&1 | grep password | awk '{print $2}'`
 316     export HOSTIP=$(echo $PROMETHEUS_HOSTNAME | sed "s/prometheus.//" | sed "s/.xip.io//")
 317
 318     rm robot-systest.cfg
 319     for line in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg ; done
 320     cat << EOF >> robot-systest.cfg
 321 VIM_TARGET=osm
 322 VIM_MGMT_NET=osm-ext
 323 ENVIRONMENTS_FOLDER=environments
 324 PACKAGES_FOLDER=/robot-systest/osm-packages
 325 OS_CLOUD=openstack
 326 LC_ALL=C.UTF-8
 327 LANG=C.UTF-8
 328 EOF
 329
 330     cat << EOF > robot.etc.hosts
 331 127.0.0.1           localhost
 332 ${HOSTIP}      prometheus.${HOSTIP}.xip.io nbi.${HOSTIP}.xip.io
 333 EOF
 334     cat << EOF > clouds.yaml
 335 clouds:
 336   openstack:
 337     auth:
 338       auth_url: $OS_AUTH_URL
 339       project_name: $OS_PROJECT_NAME
 340       username: $OS_USERNAME
 341       password: $OS_PASSWORD
 342       user_domain_name: $OS_USER_DOMAIN_NAME
 343       project_domain_name: $OS_PROJECT_DOMAIN_NAME
 344 EOF
 345
 346     VIM_AUTH_URL=$(osm vim-show osm | grep vim_url | awk '{print $4}' | tr -d \")
 347     if [ ! -z ${VIM_AUTH_URL} -a "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ] ; then
 348         echo "Deleting existing VIM osm as auth URLs have changed"
 349         osm vim-delete osm
 350     fi
 351
 352     if ! osm vim-show osm &> /dev/null ; then
 353         echo "Creating VIM osm"
 354         osm vim-create --name osm --user "$OS_USERNAME" --password "$OS_PASSWORD" \
 355         --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
 356         --config='{use_floating_ip: True,
 357                    management_network_name: osm-ext}'
 358     fi
 359
 360     if [ ! -z $ROBOT_LOCAL ] ; then
 361         LOCAL_MOUNT_1="/robot-systest/lib"
 362         LOCAL_MOUNT_2="/robot-systest/resources"
 363         LOCAL_MOUNT_3="/robot-systest/testsuite"
 364     else
 365         LOCAL_MOUNT_1="/tmp/lib"
 366         LOCAL_MOUNT_2="/tmp/resources"
 367         LOCAL_MOUNT_3="/tmp/testsuite"
 368     fi
 369
 370     mkdir -p reports
 371
 372     docker run -ti --entrypoint /bin/bash \
 373         --env OSM_HOSTNAME=${OSM_HOSTNAME} \
 374         --env PROMETHEUS_HOSTNAME=${PROMETHEUS_HOSTNAME} \
 375         --env PROMETHEUS_PORT=${PROMETHEUS_PORT} \
 376         --env JUJU_PASSWORD=${JUJU_PASSWORD} \
 377         --env HOSTIP=${HOSTIP} \
 378         --env-file robot-systest.cfg \
 379         -v "$(pwd)/robot.etc.hosts":/etc/hosts \
 380         -v ~/.osm/microk8s-config.yaml:/root/.kube/config \
 381         -v "$(pwd)/clouds.yaml":/etc/openstack/clouds.yaml \
 382         -v "$(pwd)/reports:"/robot-systest/reports \
 383         -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
 384         -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
 385         -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
 386         opensourcemano/tests:devel
 387 }
 388
 389 function update_osm_module() {
 390     MODULES=$(_find_module_dockerfile $1)
 391     for MODULE in ${MODULES} ; do
 392         MODULE=${MODULE,,}
 393         echo "Updating ${MODULE}"
 394         juju attach-resource ${MODULE} image=localhost:32000/opensourcemano/${MODULE}:devel
 395     done
 396 }
 397
 398 if [ "$0" != "$BASH_SOURCE" ]; then
 399
 400     _osm_local_build()
 401     {
 402         OPTIONS=$(show_help | sed '0,/^OPTIONS:$/d' | awk '{print $1}')
 403         COMPREPLY=($(compgen -W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
 404     }
 405
 406     THIS_SCRIPT="$(basename ${BASH_SOURCE[0]})"
 407     echo "Setting up bash completion for ${THIS_SCRIPT}"
 408     complete -F _osm_local_build "${THIS_SCRIPT}"
 409 else
 410     check_arguments $@
 411
 412     eval "${INSTALL_HTTPD}"
 413     eval "${INSTALL_LOCAL_REGISTRY}"
 414     eval "${INSTALL_MICROSTACK}"
 415     eval "${STAGE_2}"
 416     eval "${STAGE_3}"
 417     eval "${REGISTRY_PUSH}"
 418     eval "${INSTALL_OSM}"
 419     eval "${UPDATE_INSTALL}"
 420     eval "${START_ROBOT}"
 421 fi