tools/local-build.sh

   1 #!/bin/bash
   2 #######################################################################################
   3 # Copyright ETSI Contributors and Others.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #######################################################################################
  18
  19 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
  20 HTTPDDIR="$( cd "${HOME}/snap/qhttp/common" &> /dev/null && pwd )"
  21 HTTPPORT=11480
  22 KUBECFG="~/.osm/microk8s-config.yaml"
  23 NO_CACHE=""
  24 OPENSTACKRC="/var/snap/microstack/common/etc/microstack.rc"
  25 REGISTRY="localhost:32000"
  26 ROOTDIR="$( cd "${DIR}/../../" &> /dev/null && pwd)"
  27 OSM_TESTS_IMAGE_TAG="devel"
  28
  29 function check_arguments(){
  30     while [ $# -gt 0 ] ; do
  31         case $1 in
  32             --debug) set -x ;;
  33             --help | -h) show_help && exit 0 ;;
  34             --httpddir) HTTPDIR="$2" && shift;;
  35             --install-local-registry) 'install_local_registry' ;;
  36             --install-microstack) 'install_microstack' ;;
  37             --install-qhttpd) INSTALL_HTTPD='install_qhttpd' ;;
  38             --kubecfg) KUBECFG="$2" && shift ;;
  39             --module) TARGET_MODULE="$2" && shift;;
  40             --no-cache) NO_CACHE="--no-cache" ;;
  41             --openstackrc) OPENSTACKRC="$2" && shift ;;
  42             --registry) REGISTRY="$2" && shift;;
  43             --robot-local-mounts) ROBOT_LOCAL=YES ;;
  44             --run-tests) TESTS=YES ;;
  45             --vim-vca) VIM_VCA="$2" && shift;;
  46             --osm-tests-image-tag) OSM_TESTS_IMAGE_TAG="$2" && shift;;
  47             stage-2) STAGE_2='stage_2 ${TARGET_MODULE}' ;;
  48             stage-3) STAGE_3='stage_3 ${TARGET_MODULE}' ;;
  49             registry-push) REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  50             install-osm) INSTALL_OSM='install_osm' ;;
  51             start-robot) START_ROBOT='start_robot' ;;
  52             update-install) UPDATE_INSTALL='update_osm_module ${TARGET_MODULE}'
  53                  REGISTRY_PUSH='local_registry_push ${TARGET_MODULE}' ;;
  54             *)  echo "Unknown option $1"
  55                 show_help
  56                 exit 1;;
  57         esac
  58         shift
  59     done
  60 }
  61
  62 function show_help() {
  63     cat << EOF
  64 Usage: $0 [OPTIONS]
  65 Perform a local build and potential installation of OSM from sources, using the
  66 same process as Jenkins.
  67
  68 OPTIONS:
  69   --help                        display this help message
  70   --debug                       enable set -x for this script
  71   --install-local-registry     install and enable Microk8s local registry on port 32000
  72   --install-microstack          install Microstack and configure to run robot tests
  73   --install-qhttpd              install QHTTPD as an HTTP server on port ${HTTPPORT}
  74   --kubecfg                     path to kubecfg.yaml (uses Charmed OSM by default)
  75   --no-cache                    do not use any cache when building docker images
  76   --module                      only build this comma delimited list of modules
  77   --openstackrc                 path to Openstack RC file (uses Microstack by default)
  78   --registry                    use this alternate docker registry
  79   --run-tests                   run stage 2 tests
  80   --vim-vca                     name of the a vca registered in OSM to use in the VIM account
  81   --osm-tests-image-tag         tag to be used in the osm/tests docker image
  82   stage-2                       run the stage 2 build
  83   stage-3                       run the stage 3 build
  84   registry-push                 push to the local registry
  85   install-osm                   perform full installation of Charmed OSM from registry
  86   start-robot                   start the Robot test container and leave you at prompt
  87   update-install                update Charmed OSM with new module container
  88 EOF
  89 }
  90
  91 function print_section() {
  92         echo "$@"
  93 }
  94
  95 function install_local_registry() {
  96     sudo snap install microk8s --classic
  97     microk8s status --wait-ready
  98     microk8s.enable registry
  99 }
 100
 101 function install_microstack() {
 102     sudo snap install microstack --devmode --edge
 103     sudo snap set microstack config.network.ports.dashboard=8080
 104     sudo microstack.init --auto --control
 105
 106     sudo snap alias microstack.openstack openstack
 107     . /var/snap/microstack/common/etc/microstack.rc
 108
 109     for i in $(microstack.openstack security group list | awk '/default/{ print $2 }'); do
 110         microstack.openstack security group rule create $i --protocol icmp --remote-ip 0.0.0.0/0
 111         microstack.openstack security group rule create $i --protocol tcp --remote-ip 0.0.0.0/0
 112     done
 113
 114     microstack.openstack network create --enable --no-share osm-ext
 115     microstack.openstack subnet create osm-ext-subnet --network osm-ext --dns-nameserver 8.8.8.8 \
 116               --subnet-range 172.30.0.0/24
 117     microstack.openstack router create external-router
 118     microstack.openstack router add subnet external-router osm-ext-subnet
 119     microstack.openstack router set --external-gateway external external-router
 120
 121     curl -L https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img \
 122         | microstack.openstack image create --public --container-format=bare \
 123          --disk-format=qcow2 cirros-0.3.5-x86_64-disk.img
 124     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 125         | microstack.openstack image create --public --container-format=bare \
 126          --disk-format=qcow2 ubuntu16.04
 127     curl https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img \
 128         | microstack.openstack image create --public --container-format=bare \
 129          --disk-format=qcow2 US1604
 130     curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img \
 131         | microstack.openstack image create --public --container-format=bare \
 132          --disk-format=qcow2 ubuntu18.04
 133     curl https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img \
 134         | microstack.openstack image create --public --container-format=bare \
 135          --disk-format=qcow2 ubuntu20.04
 136 }
 137
 138 function install_qhttpd() {
 139     sudo snap install qhttp
 140     EXISTING_PID=$(ps auxw | grep "http.server 11480" | grep -v grep | awk '{print $2}')
 141     if [ ! -z $EXISTING_PID ] ; then
 142         kill $EXISTING_PID
 143     fi
 144     qhttp -p ${HTTPPORT} &
 145 }
 146
 147 function stage_2() {
 148     print_section "Performing Stage 2"
 149     MODULES="common devops IM LCM MON N2VC NBI NG-UI osmclient PLA POL RO tests"
 150     if [ ! -z ${1} ] ; then
 151         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 152         for MODULE in ${POSSIBLE_MODULES}; do
 153             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 154                 echo "Unknown stage 2 module ${MODULE}"
 155                 echo "Must be one of ${MODULES}"
 156                 exit 1
 157             fi
 158         done
 159         MODULES=${POSSIBLE_MODULES}
 160     else
 161         print_section "Cleaning HTTP Directory for full build"
 162         rm -fv ${HTTPDDIR}/*.deb
 163     fi
 164
 165     for MODULE in ${MODULES} ; do
 166         cd "${ROOTDIR}"
 167         if [ ! -d ${MODULE} ] ; then
 168             echo "Directory ${ROOTDIR}/${MODULE} does not exist"
 169             exit 1
 170         fi
 171         print_section "Building ${MODULE}"
 172         cd ${MODULE}
 173         find . -name '*.deb' -exec rm -v {} \;
 174         docker build ${NO_CACHE} -t ${MODULE,,}-stage2 .
 175
 176         STAGES="stage-build.sh"
 177         if [ ! -z $TESTS ] ; then
 178             STAGES="stage-test.sh ${STAGES}"
 179         fi
 180         for STAGE in $STAGES ; do
 181             docker run -ti \
 182                 -v "$(pwd):/build" \
 183                 -w /build \
 184                 ${MODULE,,}-stage2 \
 185                 bash -c "groupadd -o -g $(id -g) -r $USER ;
 186                 useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
 187                 runuser $USER -c devops-stages/${STAGE}"
 188             if [ $? -ne 0 ] ; then
 189             print_section "Failed to build ${MODULE}"
 190                 exit 1
 191             fi
 192         done
 193
 194         find . -name '*.deb' -exec mv -v {} ${HTTPDDIR}/ \;
 195     done
 196 }
 197
 198 function _find_module_dockerfile() {
 199     cd "${ROOTDIR}/devops/docker"
 200     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 201     if [ ! -z ${1} ] ; then
 202         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 203         for MODULE in ${POSSIBLE_MODULES}; do
 204             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 205                 echo "Unknown stage 3 module ${MODULE}"
 206                 echo "Must be one of ${MODULES}"
 207                 exit 1
 208             fi
 209         done
 210         echo ${POSSIBLE_MODULES}
 211     else
 212         echo ${MODULES}
 213     fi
 214 }
 215
 216 function stage_3() {
 217     print_section "Performing Stage 3"
 218     MODULES=$(_find_module_dockerfile $1)
 219     BUILD_ARGS=""
 220     HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
 221     for file in ~/snap/qhttp/common/*.deb ; do
 222         file=`basename ${file}`
 223         name=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g"`;
 224         name=${name^^}_URL
 225         BUILD_ARGS="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
 226         echo Added ${name} as http://$HOSTIP:${HTTPPORT}/$file
 227     done
 228
 229     for MODULE in ${MODULES} ; do
 230         cd "${ROOTDIR}/devops/docker"
 231         if [ ! -d ${MODULE} ] ; then
 232             echo "Directory ${ROOTDIR}/${MODULE} does not exist"
 233             exit 1
 234         fi
 235         print_section "Building ${MODULE}"
 236         cd ${MODULE}
 237         MODULE=${MODULE,,}
 238         docker build ${NO_CACHE} -t opensourcemano/${MODULE}:devel ${BUILD_ARGS} .
 239         if [ $? -ne 0 ] ; then
 240         print_section "Failed to build ${MODULE}"
 241             exit 1
 242         fi
 243     done
 244 }
 245
 246 function local_registry_push() {
 247     print_section "Pushing to local registry"
 248     cd "${ROOTDIR}/devops/docker"
 249     MODULES=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
 250     if [ ! -z ${1} ] ; then
 251         POSSIBLE_MODULES=$(echo ${1} | sed "s/,/ /g")
 252         for MODULE in ${POSSIBLE_MODULES}; do
 253             echo "${MODULE}"
 254             if ! echo "${MODULES}" | grep -q "${MODULE}" ; then
 255                 echo "Unknown stage 3 module ${MODULE}"
 256                 echo "Must be one of ${MODULES}"
 257                 exit 1
 258             fi
 259         done
 260         MODULES=${POSSIBLE_MODULES}
 261     fi
 262     for MODULE in ${MODULES} ; do
 263         MODULE=${MODULE,,}
 264         docker tag opensourcemano/${MODULE}:devel ${REGISTRY}/opensourcemano/${MODULE}:devel
 265         docker push ${REGISTRY}/opensourcemano/${MODULE}:devel
 266     done
 267 }
 268
 269 function install_osm() {
 270     cd "${ROOTDIR}/devops/installers"
 271     VCA=""
 272     if juju controllers 2>/dev/null| grep osm-vca ; then
 273         VCA="--vca osm-vca"
 274     fi
 275     ./charmed_install.sh --registry localhost:32000  --tag devel ${VCA}
 276 }
 277
 278 function start_robot() {
 279     mkdir -p "${ROOTDIR}/tests/local"
 280     cd "${ROOTDIR}/tests/local"
 281
 282     . ${OPENSTACKRC}
 283
 284     # Workaround for microstack auth URL
 285     if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
 286         export OS_AUTH_URL=${OS_AUTH_URL}/v3
 287     fi
 288
 289     export OSM_HOSTNAME=$(juju config -m osm nbi site_url | sed "s/http.*\?:\/\///"):443
 290     export PROMETHEUS_HOSTNAME=$(juju config -m osm prometheus site_url | sed "s/http.*\?:\/\///")
 291     export PROMETHEUS_PORT=80
 292     export JUJU_PASSWORD=`juju gui 2>&1 | grep password | awk '{print $2}'`
 293     export HOSTIP=$(echo $PROMETHEUS_HOSTNAME | sed "s/prometheus.//" | sed "s/.nip.io//")
 294
 295     rm robot-systest.cfg
 296     for line in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg ; done
 297     cat << EOF >> robot-systest.cfg
 298 VIM_TARGET=osm
 299 VIM_MGMT_NET=osm-ext
 300 ENVIRONMENTS_FOLDER=environments
 301 PACKAGES_FOLDER=/robot-systest/osm-packages
 302 OS_CLOUD=openstack
 303 LC_ALL=C.UTF-8
 304 LANG=C.UTF-8
 305 EOF
 306
 307     cat << EOF > robot.etc.hosts
 308 127.0.0.1           localhost
 309 ${HOSTIP}      prometheus.${HOSTIP}.nip.io nbi.${HOSTIP}.nip.io
 310 EOF
 311     cat << EOF > clouds.yaml
 312 clouds:
 313   openstack:
 314     auth:
 315       auth_url: $OS_AUTH_URL
 316       project_name: $OS_PROJECT_NAME
 317       username: $OS_USERNAME
 318       password: $OS_PASSWORD
 319       user_domain_name: $OS_USER_DOMAIN_NAME
 320       project_domain_name: $OS_PROJECT_DOMAIN_NAME
 321 EOF
 322
 323     VIM_AUTH_URL=$(osm vim-show osm | grep vim_url | awk '{print $4}' | tr -d \")
 324     if [[ ! -z ${VIM_AUTH_URL} && "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ]] ; then
 325         echo "Deleting existing VIM osm as auth URLs have changed"
 326         osm vim-delete osm
 327     fi
 328
 329     if ! osm vim-show osm &> /dev/null ; then
 330         echo "Creating VIM osm"
 331         if [ -v VIM_VCA ]; then
 332            VCA_OPT="--vca $VIM_VCA"
 333         fi
 334         osm vim-create --name osm $VCA_OPT --user "$OS_USERNAME" --password "$OS_PASSWORD" \
 335         --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
 336         --config='{use_floating_ip: True,
 337                    management_network_name: osm-ext}'
 338     fi
 339
 340     if [ ! -z $ROBOT_LOCAL ] ; then
 341         LOCAL_MOUNT_1="/robot-systest/lib"
 342         LOCAL_MOUNT_2="/robot-systest/resources"
 343         LOCAL_MOUNT_3="/robot-systest/testsuite"
 344     else
 345         LOCAL_MOUNT_1="/tmp/lib"
 346         LOCAL_MOUNT_2="/tmp/resources"
 347         LOCAL_MOUNT_3="/tmp/testsuite"
 348     fi
 349
 350     mkdir -p reports
 351
 352     docker run -ti --entrypoint /bin/bash \
 353         --env OSM_HOSTNAME=${OSM_HOSTNAME} \
 354         --env PROMETHEUS_HOSTNAME=${PROMETHEUS_HOSTNAME} \
 355         --env PROMETHEUS_PORT=${PROMETHEUS_PORT} \
 356         --env JUJU_PASSWORD=${JUJU_PASSWORD} \
 357         --env HOSTIP=${HOSTIP} \
 358         --env-file robot-systest.cfg \
 359         -v "$(pwd)/robot.etc.hosts":/etc/hosts \
 360         -v ~/.osm/microk8s-config.yaml:/root/.kube/config \
 361         -v "$(pwd)/clouds.yaml":/etc/openstack/clouds.yaml \
 362         -v "$(pwd)/reports:"/robot-systest/reports \
 363         -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
 364         -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
 365         -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
 366         opensourcemano/tests:$OSM_TESTS_IMAGE_TAG
 367 }
 368
 369 function update_osm_module() {
 370     MODULES=$(_find_module_dockerfile $1)
 371     for MODULE in ${MODULES} ; do
 372         MODULE=${MODULE,,}
 373         echo "Updating ${MODULE}"
 374         juju attach-resource ${MODULE} image=localhost:32000/opensourcemano/${MODULE}:devel
 375     done
 376 }
 377
 378 if [ "$0" != "$BASH_SOURCE" ]; then
 379
 380     _osm_local_build()
 381     {
 382         OPTIONS=$(show_help | sed '0,/^OPTIONS:$/d' | awk '{print $1}')
 383         COMPREPLY=($(compgen -W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
 384     }
 385
 386     THIS_SCRIPT="$(basename ${BASH_SOURCE[0]})"
 387     echo "Setting up bash completion for ${THIS_SCRIPT}"
 388     complete -F _osm_local_build "${THIS_SCRIPT}"
 389 else
 390     check_arguments $@
 391
 392     eval "${INSTALL_HTTPD}"
 393     eval "${INSTALL_LOCAL_REGISTRY}"
 394     eval "${INSTALL_MICROSTACK}"
 395     eval "${STAGE_2}"
 396     eval "${STAGE_3}"
 397     eval "${REGISTRY_PUSH}"
 398     eval "${INSTALL_OSM}"
 399     eval "${UPDATE_INSTALL}"
 400     eval "${START_ROBOT}"
 401 fi