tools/local-build.sh

   1 #!/bin/bash
   2 #######################################################################################
   3 # Copyright ETSI Contributors and Others.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #######################################################################################
  18
  19 APT_PROXY=""
  20 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
  21 HTTPDDIR="${HOME}/.osm/httpd"
  22 HTTPPORT=8000
  23 KUBECFG="~/.osm/microk8s-config.yaml"
  24 NO_CACHE=""
  25 OPENSTACKRC="/var/snap/microstack/common/etc/microstack.rc"
  26 REGISTRY="localhost:32000"
  27 ROOTDIR="$( cd "${DIR}/../../" &> /dev/null && pwd)"
  28 DEVEL_TAG="devel"
  29 OSM_TESTS_IMAGE_TAG="devel"
  30
  31 function check_arguments(){
  32     while [ $# -gt 0 ] ; do
  33         case $1 in
  34             --debug) set -x ;;
  35             --apt-proxy) APT_PROXY="$2" && shift ;;
  36             --devel-tag) DEVEL_TAG="$2" && shift ;;
  37             --help | -h) show_help && exit 0 ;;
  38             --httpddir) HTTPDDIR="$2" && shift;;
  39             --install-local-registry) 'install_local_registry' ;;
  40             --install-microstack) 'install_microstack' ;;
  41             --install-qhttpd) INSTALL_HTTPD='install_qhttpd' ;;
  42             --run-httpserver) INSTALL_HTTPD='run_httpserver' ;;
  43             --kubecfg) KUBECFG="$2" && shift ;;
  44             --module) TARGET_MODULE="$2" && shift;;
  45             --no-cache) NO_CACHE="--no-cache" ;;
  46             --openstackrc) OPENSTACKRC="$2" && shift ;;
  47             --registry) REGISTRY="$2" && shift;;
  48             --robot-local-mounts) ROBOT_LOCAL=YES ;;
  49             --run-tests) TESTS=YES ;;
  50             --vim-vca) VIM_VCA="$2" && shift;;
  51             --osm-tests-image-tag) OSM_TESTS_IMAGE_TAG="$2" && shift;;
  52             stage-2) STAGE_2='stage_2 ${TARGET_MODULE}' ;;
  53             stage-3) STAGE_3='stage_3 ${TARGET_MODULE}' ;;
  54             registry-push) REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  55             install-osm) INSTALL_OSM='install_osm' ;;
  56             start-robot) START_ROBOT='start_robot' ;;
  57             update-install) UPDATE_INSTALL='update_osm_module ${TARGET_MODULE}'
  58                  REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  59             *)  echo "Unknown option $1"
  60                 show_help
  61                 exit 1;;
  62         esac
  63         shift
  64     done
  65 }
  66
  67 function show_help() {
  68     cat << EOF
  69 Usage: $0 [OPTIONS]
  70 Perform a local build and potential installation of OSM from sources, using the
  71 same process as Jenkins.
  72
  73 OPTIONS:
  74   --help                        display this help message
  75   --apt-proxy                   provide an apt proxy to docker build steps
  76   --debug                       enable set -x for this script
  77   --install-local-registry      install and enable Microk8s local registry on port 32000
  78   --install-microstack          install Microstack and configure to run robot tests
  79   --install-qhttpd              (deprecated, use --run-httpserver instead) install QHTTPD as an HTTP server on port ${HTTPPORT}
  80   --run-httpserver              run HTTP server on port ${HTTPPORT}
  81   --kubecfg                     path to kubecfg.yaml (uses Charmed OSM by default)
  82   --no-cache                    do not use any cache when building docker images
  83   --module                      only build this comma delimited list of modules
  84   --openstackrc                 path to Openstack RC file (uses Microstack by default)
  85   --registry                    use this alternate docker registry
  86   --run-tests                   run stage 2 tests
  87   --vim-vca                     name of the a vca registered in OSM to use in the VIM account
  88   --osm-tests-image-tag         tag to be used in the osm/tests docker image
  89   stage-2                       run the stage 2 build
  90   stage-3                       run the stage 3 build
  91   registry-push                 push to the local registry
  92   install-osm                   perform full installation of Charmed OSM from registry
  93   start-robot                   start the Robot test container and leave you at prompt
  94   update-install                update Charmed OSM with new module container
  95
  96 A typical use could be the following:
  97
  98 Let's assume that we have different repos cloned in the folder workspace:
  99
 100   cd workspace
 101   git clone https://osm.etsi.org/gerrit/osm/devops
 102   git clone https://osm.etsi.org/gerrit/osm/NBI
 103   git clone https://osm.etsi.org/gerrit/osm/LCM
 104   git clone "https://osm.etsi.org/gerrit/osm/RO
 105   git clone "https://osm.etsi.org/gerrit/osm/common
 106   git clone "https://osm.etsi.org/gerrit/osm/IM
 107   git clone "https://osm.etsi.org/gerrit/osm/N2VC
 108
 109 First we run a light HTTP server to serve the artifacts:
 110
 111   devops/tools/local-build.sh --run-httpserver
 112
 113 Then we generate the artifacts (debian packages) for the different repos: common, IM, N2VC, RO, LCM, NBI
 114
 115   devops/tools/local-build.sh --module common,IM,N2VC,RO,LCM,NBI stage-2
 116
 117 Then new docker images are generated locally with the tag "devel" (e.g.: opensourcemano/lcm:devel):
 118
 119   devops/tools/local-build.sh --module RO,LCM,NBI stage-3
 120
 121 Finally, the deployment of OSM will have to be updated to use the new docker images.
 122
 123 EOF
 124 }
 125
 126 function print_section() {
 127         echo "$@"
 128 }
 129
 130 function install_local_registry() {
 131     sudo snap install microk8s --classic
 132     microk8s status --wait-ready
 133     microk8s.enable registry
 134 }
 135
 136 function install_microstack() {
 137     sudo snap install microstack --devmode --edge
 138     sudo snap set microstack config.network.ports.dashboard=8080
 139     sudo microstack.init --auto --control
 140
 141     sudo snap alias microstack.openstack openstack
 142     . /var/snap/microstack/common/etc/microstack.rc
 143
 144     for i in $(microstack.openstack security group list | awk '/default/{ print $2 }'); do
 145         microstack.openstack security group rule create $i --protocol icmp --remote-ip 0.0.0.0/0
 146         microstack.openstack security group rule create $i --protocol tcp --remote-ip 0.0.0.0/0
 147     done
 148
 149     microstack.openstack network create --enable --no-share osm-ext
 150     microstack.openstack subnet create osm-ext-subnet --network osm-ext --dns-nameserver 8.8.8.8 \
 151               --subnet-range 172.30.0.0/24
 152     microstack.openstack router create external-router
 153     microstack.openstack router add subnet external-router osm-ext-subnet
 154     microstack.openstack router set --external-gateway external external-router
 155
 156     curl -L https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img \
 157         | microstack.openstack image create --public --container-format=bare \
 158          --disk-format=qcow2 cirros-0.3.5-x86_64-disk.img
 159     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 160         | microstack.openstack image create --public --container-format=bare \
 161          --disk-format=qcow2 ubuntu16.04
 162     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 163         | microstack.openstack image create --public --container-format=bare \
 164          --disk-format=qcow2 US1604
 165     curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img \
 166         | microstack.openstack image create --public --container-format=bare \
 167          --disk-format=qcow2 ubuntu18.04
 168     curl https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img \
 169         | microstack.openstack image create --public --container-format=bare \
 170          --disk-format=qcow2 ubuntu20.04
 171 }
 172
 173 function create_httpddir() {
 174     mkdir -p ${HTTPDDIR}
 175 }
 176
 177 function install_qhttpd() {
 178     sudo snap install qhttp
 179     EXISTING_PID=$(ps auxw | grep "http.server $HTTPPORT" | grep -v grep | awk '{print $2}')
 180     if [ ! -z $EXISTING_PID ] ; then
 181         kill $EXISTING_PID
 182     fi
 183     nohup qhttp -p ${HTTPPORT} &
 184 }
 185
 186 function run_httpserver() {
 187     EXISTING_PID=$(ps auxw | grep "http.server $HTTPPORT" | grep -v grep | awk '{print $2}')
 188     if [ ! -z $EXISTING_PID ] ; then
 189         kill $EXISTING_PID
 190     fi
 191     nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null &
 192 }
 193
 194 function run_httpserver() {
 195     EXISTING_PID=$(ps auxw | grep "http.server $HTTPPORT" | grep -v grep | awk '{print $2}')
 196     if [ ! -z $EXISTING_PID ] ; then
 197         kill $EXISTING_PID
 198     fi
 199     nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null &
 200 }
 201
 202 function stage_2() {
 203     print_section "Performing Stage 2"
 204     MODULES="common devops IM LCM MON N2VC NBI NG-UI NG-SA osmclient PLA POL RO tests"
 205     if [ ! -z ${1} ] ; then
 206         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 207         for MODULE in ${POSSIBLE_MODULES}; do
 208             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 209                 echo "Unknown stage 2 module ${MODULE}"
 210                 echo "Must be one of ${MODULES}"
 211                 exit 1
 212             fi
 213         done
 214         MODULES=${POSSIBLE_MODULES}
 215     else
 216         print_section "Cleaning HTTP Directory for full build"
 217         rm -fv ${HTTPDDIR}/*.deb
 218     fi
 219
 220     for MODULE in ${MODULES} ; do
 221         cd "${ROOTDIR}"
 222         if [ ! -d ${MODULE} ] ; then
 223             echo "Directory ${ROOTDIR}/${MODULE} does not exist"
 224             exit 1
 225         fi
 226         print_section "Building ${MODULE}"
 227         cd ${MODULE}
 228         find . -name '*.deb' -exec rm -v {} \;
 229
 230         BUILD_ARGS=""
 231         if [ ! -z $APT_PROXY ] ; then
 232             BUILD_ARGS="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
 233         fi
 234         docker build ${NO_CACHE} ${BUILD_ARGS} -t ${MODULE,,}-stage2 .
 235
 236         STAGES="stage-build.sh"
 237         if [ ! -z $TESTS ] ; then
 238             STAGES="stage-test.sh ${STAGES}"
 239         fi
 240         for STAGE in $STAGES ; do
 241             docker run -i \
 242                 -v "$(pwd):/build" \
 243                 -w /build \
 244                 ${MODULE,,}-stage2 \
 245                 bash -c "groupadd -o -g $(id -g) -r $USER ;
 246                 useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
 247                 runuser $USER -c devops-stages/${STAGE}"
 248             if [ $? -ne 0 ] ; then
 249             print_section "Failed to build ${MODULE}"
 250                 exit 1
 251             fi
 252         done
 253         for file in `find . -name '*.deb'` ; do
 254             name=`basename ${file} | cut -d_ -f1`;
 255             rm -v ~/snap/qhttp/common/${name}*
 256             cp -v $file ~/snap/qhttp/common/${name}_$(date "+%H%M%S").deb
 257             rm -v $file
 258         done
 259     done
 260 }
 261
 262 function _find_module_dockerfile() {
 263     cd "${ROOTDIR}/devops/docker"
 264     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 265     if [ ! -z ${1} ] ; then
 266         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 267         for MODULE in ${POSSIBLE_MODULES}; do
 268             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 269                 echo "Unknown stage 3 module ${MODULE}"
 270                 echo "Must be one of ${MODULES}"
 271                 exit 1
 272             fi
 273         done
 274         echo ${POSSIBLE_MODULES}
 275     else
 276         echo ${MODULES}
 277     fi
 278 }
 279
 280 function stage_3() {
 281     print_section "Performing Stage 3"
 282     MODULES=$(_find_module_dockerfile $1)
 283     BUILD_ARGS=""
 284     if [ ! -z $APT_PROXY ] ; then
 285         BUILD_ARGS="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
 286     fi
 287
 288     HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
 289     [ -z "$DEFAULT_IF" ] && DEFAULT_IF=$(ip route list|awk '$1=="default" {print $5; exit}')
 290     [ -z "$DEFAULT_IF" ] && DEFAULT_IF=$(route -n |awk '$1~/^0.0.0.0/ {print $8; exit}')
 291     DEFAULT_IP=$(ip -o -4 a s ${DEFAULT_IF} |awk '{split($4,a,"/"); print a[1]; exit}')
 292     HOSTIP=${HOSTIP:=${DEFAULT_IP}}
 293     echo $HOSTIP
 294
 295     for file in ${HTTPDDIR}/*.deb ; do
 296         file=`basename ${file}`
 297         name=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g" | sed "s/.deb//"`;
 298         name=${name^^}_URL
 299         BUILD_ARGS="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
 300         echo Added ${name} as http://$HOSTIP:${HTTPPORT}/$file
 301     done
 302
 303     for MODULE in ${MODULES} ; do
 304         cd "${ROOTDIR}/devops/docker"
 305         if [ ! -d ${MODULE} ] ; then
 306             echo "Directory ${ROOTDIR}/devops/docker/${MODULE} does not exist"
 307             exit 1
 308         fi
 309         print_section "Building ${MODULE}"
 310         cd ${MODULE}
 311         MODULE=${MODULE,,}
 312         docker build ${NO_CACHE} -t opensourcemano/${MODULE}:${DEVEL_TAG} ${BUILD_ARGS} .
 313         if [ $? -ne 0 ] ; then
 314         print_section "Failed to build ${MODULE}"
 315             exit 1
 316         fi
 317     done
 318 }
 319
 320 function local_registry_push() {
 321     print_section "Pushing to local registry"
 322     cd "${ROOTDIR}/devops/docker"
 323     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 324     if [ ! -z ${1} ] ; then
 325         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 326         for MODULE in ${POSSIBLE_MODULES}; do
 327             echo "${MODULE}"
 328             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 329                 echo "Unknown stage 3 module ${MODULE}"
 330                 echo "Must be one of ${MODULES}"
 331                 exit 1
 332             fi
 333         done
 334         MODULES=${POSSIBLE_MODULES}
 335     fi
 336     for MODULE in ${MODULES} ; do
 337         MODULE=${MODULE,,}
 338         docker tag opensourcemano/${MODULE}:${DEVEL_TAG} ${REGISTRY}/opensourcemano/${MODULE}:${DEVEL_TAG}
 339         docker push ${REGISTRY}/opensourcemano/${MODULE}:${DEVEL_TAG}
 340     done
 341 }
 342
 343 function install_osm() {
 344     cd "${ROOTDIR}/devops/installers"
 345     VCA=""
 346     if juju controllers 2>/dev/null| grep osm-vca ; then
 347         VCA="--vca osm-vca"
 348     fi
 349     ./charmed_install.sh --registry localhost:32000  --tag ${DEVEL_TAG} ${VCA}
 350 }
 351
 352 function start_robot() {
 353     mkdir -p "${ROOTDIR}/tests/local"
 354     cd "${ROOTDIR}/tests/local"
 355
 356     . ${OPENSTACKRC}
 357
 358     # Workaround for microstack auth URL
 359     if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
 360         export OS_AUTH_URL=${OS_AUTH_URL}/v3
 361     fi
 362
 363     export OSM_HOSTNAME=$(juju config -m osm nbi site_url | sed "s/http.*\?:\/\///"):443
 364     export PROMETHEUS_HOSTNAME=$(juju config -m osm prometheus site_url | sed "s/http.*\?:\/\///")
 365     export PROMETHEUS_PORT=80
 366     export JUJU_PASSWORD=`juju gui 2>&1 | grep password | awk '{print $2}'`
 367     export HOSTIP=$(echo $PROMETHEUS_HOSTNAME | sed "s/prometheus.//" | sed "s/.nip.io//")
 368
 369     rm robot-systest.cfg
 370     for line in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg ; done
 371     cat << EOF >> robot-systest.cfg
 372 VIM_TARGET=osm
 373 VIM_MGMT_NET=osm-ext
 374 ENVIRONMENTS_FOLDER=environments
 375 PACKAGES_FOLDER=/robot-systest/osm-packages
 376 OS_CLOUD=openstack
 377 LC_ALL=C.UTF-8
 378 LANG=C.UTF-8
 379 EOF
 380
 381     cat << EOF > robot.etc.hosts
 382 127.0.0.1           localhost
 383 ${HOSTIP}      prometheus.${HOSTIP}.nip.io nbi.${HOSTIP}.nip.io
 384 EOF
 385     cat << EOF > clouds.yaml
 386 clouds:
 387   openstack:
 388     auth:
 389       auth_url: $OS_AUTH_URL
 390       project_name: $OS_PROJECT_NAME
 391       username: $OS_USERNAME
 392       password: $OS_PASSWORD
 393       user_domain_name: $OS_USER_DOMAIN_NAME
 394       project_domain_name: $OS_PROJECT_DOMAIN_NAME
 395 EOF
 396
 397     VIM_AUTH_URL=$(osm vim-show osm | grep vim_url | awk '{print $4}' | tr -d \")
 398     if [[ ! -z ${VIM_AUTH_URL} && "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ]] ; then
 399         echo "Deleting existing VIM osm as auth URLs have changed"
 400         osm vim-delete osm
 401     fi
 402
 403     if ! osm vim-show osm &> /dev/null ; then
 404         echo "Creating VIM osm"
 405         if [ -v VIM_VCA ]; then
 406            VCA_OPT="--vca $VIM_VCA"
 407         fi
 408         osm vim-create --name osm $VCA_OPT --user "$OS_USERNAME" --password "$OS_PASSWORD" \
 409         --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
 410         --config='{use_floating_ip: True,
 411                    management_network_name: osm-ext}'
 412     fi
 413
 414     if [ ! -z $ROBOT_LOCAL ] ; then
 415         LOCAL_MOUNT_1="/robot-systest/lib"
 416         LOCAL_MOUNT_2="/robot-systest/resources"
 417         LOCAL_MOUNT_3="/robot-systest/testsuite"
 418     else
 419         LOCAL_MOUNT_1="/tmp/lib"
 420         LOCAL_MOUNT_2="/tmp/resources"
 421         LOCAL_MOUNT_3="/tmp/testsuite"
 422     fi
 423
 424     mkdir -p reports
 425
 426     docker run -ti --entrypoint /bin/bash \
 427         --env OSM_HOSTNAME=${OSM_HOSTNAME} \
 428         --env PROMETHEUS_HOSTNAME=${PROMETHEUS_HOSTNAME} \
 429         --env PROMETHEUS_PORT=${PROMETHEUS_PORT} \
 430         --env JUJU_PASSWORD=${JUJU_PASSWORD} \
 431         --env HOSTIP=${HOSTIP} \
 432         --env-file robot-systest.cfg \
 433         -v "$(pwd)/robot.etc.hosts":/etc/hosts \
 434         -v ~/.osm/microk8s-config.yaml:/root/.kube/config \
 435         -v "$(pwd)/clouds.yaml":/etc/openstack/clouds.yaml \
 436         -v "${HOME}/snap/qhttp/common"/robot-systest/reports \
 437         -v "${HOME}/snap/qhttp/common:"/robot-systest/conformance-tests/reports \
 438         -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
 439         -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
 440         -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
 441         opensourcemano/tests:$OSM_TESTS_IMAGE_TAG
 442 }
 443
 444 function update_osm_module() {
 445     MODULES=$(_find_module_dockerfile $1)
 446     for MODULE in ${MODULES} ; do
 447         MODULE=${MODULE,,}
 448         echo "Updating ${MODULE}"
 449         juju attach-resource ${MODULE} ${MODULE}-image=localhost:32000/opensourcemano/${MODULE}:${DEVEL_TAG}
 450     done
 451 }
 452
 453 if [ "$0" != "$BASH_SOURCE" ]; then
 454
 455     _osm_local_build()
 456     {
 457         OPTIONS=$(show_help | sed '0,/^OPTIONS:$/d' | awk '{print $1}')
 458         COMPREPLY=($(compgen -W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
 459     }
 460
 461     THIS_SCRIPT="$(basename ${BASH_SOURCE[0]})"
 462     echo "Setting up bash completion for ${THIS_SCRIPT}"
 463     complete -F _osm_local_build "${THIS_SCRIPT}"
 464 else
 465     check_arguments $@
 466
 467     create_httpddir
 468     eval "${INSTALL_HTTPD}"
 469     eval "${INSTALL_LOCAL_REGISTRY}"
 470     eval "${INSTALL_MICROSTACK}"
 471     eval "${STAGE_2}"
 472     eval "${STAGE_3}"
 473     eval "${REGISTRY_PUSH}"
 474     eval "${INSTALL_OSM}"
 475     eval "${UPDATE_INSTALL}"
 476     eval "${START_ROBOT}"
 477 fi