2 #######################################################################################
3 # Copyright ETSI Contributors and Others.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #######################################################################################
20 DIR
="$( cd "$
( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
21 HTTPDDIR
="$( cd "${HOME}/snap
/qhttp
/common
" &> /dev/null && pwd )"
23 KUBECFG
="~/.osm/microk8s-config.yaml"
25 OPENSTACKRC
="/var/snap/microstack/common/etc/microstack.rc"
26 REGISTRY
="localhost:32000"
27 ROOTDIR
="$( cd "${DIR}/..
/..
/" &> /dev/null && pwd)"
29 OSM_TESTS_IMAGE_TAG
="devel"
31 function check_arguments
(){
32 while [ $# -gt 0 ] ; do
35 --apt-proxy) APT_PROXY
="$2" && shift ;;
36 --devel-tag) DEVEL_TAG
="$2" && shift ;;
37 --help |
-h) show_help
&& exit 0 ;;
38 --httpddir) HTTPDIR
="$2" && shift;;
39 --install-local-registry) 'install_local_registry' ;;
40 --install-microstack) 'install_microstack' ;;
41 --install-qhttpd) INSTALL_HTTPD
='install_qhttpd' ;;
42 --kubecfg) KUBECFG
="$2" && shift ;;
43 --module) TARGET_MODULE
="$2" && shift;;
44 --no-cache) NO_CACHE
="--no-cache" ;;
45 --openstackrc) OPENSTACKRC
="$2" && shift ;;
46 --registry) REGISTRY
="$2" && shift;;
47 --robot-local-mounts) ROBOT_LOCAL
=YES
;;
48 --run-tests) TESTS
=YES
;;
49 --vim-vca) VIM_VCA
="$2" && shift;;
50 --osm-tests-image-tag) OSM_TESTS_IMAGE_TAG
="$2" && shift;;
51 stage-2
) STAGE_2
='stage_2 ${TARGET_MODULE}' ;;
52 stage-3
) STAGE_3
='stage_3 ${TARGET_MODULE}' ;;
53 registry-push
) REGISTRY_PUSH
='local_registry_push ${TARGET_MODULE}' ;;
54 install-osm
) INSTALL_OSM
='install_osm' ;;
55 start-robot
) START_ROBOT
='start_robot' ;;
56 update-install
) UPDATE_INSTALL
='update_osm_module ${TARGET_MODULE}'
57 REGISTRY_PUSH
='local_registry_push ${TARGET_MODULE}' ;;
58 *) echo "Unknown option $1"
66 function show_help
() {
69 Perform a local build and potential installation of OSM from sources, using the
70 same process as Jenkins.
73 --help display this help message
74 --apt-proxy provide an apt proxy to docker build steps
75 --debug enable set -x for this script
76 --install-local-registry install and enable Microk8s local registry on port 32000
77 --install-microstack install Microstack and configure to run robot tests
78 --install-qhttpd install QHTTPD as an HTTP server on port ${HTTPPORT}
79 --kubecfg path to kubecfg.yaml (uses Charmed OSM by default)
80 --no-cache do not use any cache when building docker images
81 --module only build this comma delimited list of modules
82 --openstackrc path to Openstack RC file (uses Microstack by default)
83 --registry use this alternate docker registry
84 --run-tests run stage 2 tests
85 --vim-vca name of the a vca registered in OSM to use in the VIM account
86 --osm-tests-image-tag tag to be used in the osm/tests docker image
87 stage-2 run the stage 2 build
88 stage-3 run the stage 3 build
89 registry-push push to the local registry
90 install-osm perform full installation of Charmed OSM from registry
91 start-robot start the Robot test container and leave you at prompt
92 update-install update Charmed OSM with new module container
94 A typical use could be the following:
96 Let's assume that we have different repos cloned in the folder workspace:
99 git clone https://osm.etsi.org/gerrit/osm/devops
100 git clone https://osm.etsi.org/gerrit/osm/NBI
101 git clone https://osm.etsi.org/gerrit/osm/LCM
102 git clone "https://osm.etsi.org/gerrit/osm/RO
103 git clone "https://osm.etsi.org/gerrit/osm/common
104 git clone "https://osm.etsi.org/gerrit/osm/IM
105 git clone "https://osm.etsi.org/gerrit/osm/N2VC
107 First we install a light HTTP server to serve the artifacts:
109 devops/tools/local-build.sh --install-qhttpd
111 Then we generate the artifacts (debian packages) for the different repos: common, IM, N2VC, RO, LCM, NBI
113 devops/tools/local-build.sh --module common,IM,N2VC,RO,LCM,NBI stage-2
115 Then new docker images are generated locally with the tag "devel" (e.g.: opensourcemano/lcm:devel):
117 devops/tools/local-build.sh --module RO,LCM,NBI stage-3
119 Finally, the deployment of OSM will have to be updated to use the new docker images.
124 function print_section
() {
128 function install_local_registry
() {
129 sudo snap
install microk8s
--classic
130 microk8s status
--wait-ready
131 microk8s.
enable registry
134 function install_microstack
() {
135 sudo snap
install microstack
--devmode --edge
136 sudo snap
set microstack config.network.ports.dashboard
=8080
137 sudo microstack.init
--auto --control
139 sudo snap
alias microstack.openstack openstack
140 .
/var
/snap
/microstack
/common
/etc
/microstack.rc
142 for i
in $
(microstack.openstack security group list |
awk '/default/{ print $2 }'); do
143 microstack.openstack security group rule create
$i --protocol icmp
--remote-ip 0.0.0.0/0
144 microstack.openstack security group rule create
$i --protocol tcp
--remote-ip 0.0.0.0/0
147 microstack.openstack network create
--enable --no-share osm-ext
148 microstack.openstack subnet create osm-ext-subnet
--network osm-ext
--dns-nameserver 8.8.8.8 \
149 --subnet-range 172.30.0.0/24
150 microstack.openstack router create external-router
151 microstack.openstack router add subnet external-router osm-ext-subnet
152 microstack.openstack router
set --external-gateway external external-router
154 curl
-L https
://github.com
/cirros-dev
/cirros
/releases
/download
/0.3.5/cirros-0.3
.5-x86_64-disk.img \
155 | microstack.openstack image create
--public --container-format=bare \
156 --disk-format=qcow2 cirros-0.3
.5-x86_64-disk.img
157 curl https
://cloud-images.ubuntu.com
/xenial
/current
/xenial-server-cloudimg-amd64-disk1.img \
158 | microstack.openstack image create
--public --container-format=bare \
159 --disk-format=qcow2 ubuntu16.04
160 curl https
://cloud-images.ubuntu.com
/xenial
/current
/xenial-server-cloudimg-amd64-disk1.img \
161 | microstack.openstack image create
--public --container-format=bare \
162 --disk-format=qcow2 US1604
163 curl https
://cloud-images.ubuntu.com
/bionic
/current
/bionic-server-cloudimg-amd64.img \
164 | microstack.openstack image create
--public --container-format=bare \
165 --disk-format=qcow2 ubuntu18.04
166 curl https
://cloud-images.ubuntu.com
/focal
/current
/focal-server-cloudimg-amd64.img \
167 | microstack.openstack image create
--public --container-format=bare \
168 --disk-format=qcow2 ubuntu20.04
171 function install_qhttpd
() {
172 sudo snap
install qhttp
173 EXISTING_PID
=$
(ps auxw |
grep "http.server $HTTPPORT" |
grep -v grep |
awk '{print $2}')
174 if [ ! -z $EXISTING_PID ] ; then
177 nohup qhttp
-p ${HTTPPORT} &
181 print_section
"Performing Stage 2"
182 MODULES
="common devops IM LCM MON N2VC NBI NG-UI NG-SA osmclient PLA POL RO tests"
183 if [ ! -z ${1} ] ; then
184 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
185 for MODULE
in ${POSSIBLE_MODULES}; do
186 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
187 echo "Unknown stage 2 module ${MODULE}"
188 echo "Must be one of ${MODULES}"
192 MODULES
=${POSSIBLE_MODULES}
194 print_section
"Cleaning HTTP Directory for full build"
195 rm -fv ${HTTPDDIR}/*.deb
198 for MODULE
in ${MODULES} ; do
200 if [ ! -d ${MODULE} ] ; then
201 echo "Directory ${ROOTDIR}/${MODULE} does not exist"
204 print_section
"Building ${MODULE}"
206 find .
-name '*.deb' -exec rm -v {} \
;
209 if [ ! -z $APT_PROXY ] ; then
210 BUILD_ARGS
="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
212 docker build
${NO_CACHE} ${BUILD_ARGS} -t ${MODULE,,}-stage2 .
214 STAGES
="stage-build.sh"
215 if [ ! -z $TESTS ] ; then
216 STAGES
="stage-test.sh ${STAGES}"
218 for STAGE
in $STAGES ; do
223 bash
-c "groupadd -o -g $(id -g) -r $USER ;
224 useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
225 runuser $USER -c devops-stages/${STAGE}"
226 if [ $?
-ne 0 ] ; then
227 print_section
"Failed to build ${MODULE}"
231 for file in `find . -name '*.deb'` ; do
232 name
=`basename ${file} | cut -d_ -f1`;
233 rm -v ~
/snap
/qhttp
/common
/${name}*
234 cp -v $file ~
/snap
/qhttp
/common
/${name}_$
(date "+%H%M%S").deb
240 function _find_module_dockerfile
() {
241 cd "${ROOTDIR}/devops/docker"
242 MODULES
=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
243 if [ ! -z ${1} ] ; then
244 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
245 for MODULE
in ${POSSIBLE_MODULES}; do
246 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
247 echo "Unknown stage 3 module ${MODULE}"
248 echo "Must be one of ${MODULES}"
252 echo ${POSSIBLE_MODULES}
259 print_section
"Performing Stage 3"
260 MODULES
=$
(_find_module_dockerfile
$1)
262 if [ ! -z $APT_PROXY ] ; then
263 BUILD_ARGS
="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
266 HOSTIP
=$
(ip
-4 addr show docker0 |
grep -Po 'inet \K[\d.]+')
267 for file in ~
/snap
/qhttp
/common
/*.deb
; do
268 file=`basename ${file}`
269 name
=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g" | sed "s/.deb//"`;
271 BUILD_ARGS
="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
272 echo Added
${name} as http
://$HOSTIP:${HTTPPORT}/$file
275 for MODULE
in ${MODULES} ; do
276 cd "${ROOTDIR}/devops/docker"
277 if [ ! -d ${MODULE} ] ; then
278 echo "Directory ${ROOTDIR}/devops/docker/${MODULE} does not exist"
281 print_section
"Building ${MODULE}"
284 docker build
${NO_CACHE} -t opensourcemano/${MODULE}:${DEVEL_TAG} ${BUILD_ARGS} .
285 if [ $?
-ne 0 ] ; then
286 print_section
"Failed to build ${MODULE}"
292 function local_registry_push
() {
293 print_section
"Pushing to local registry"
294 cd "${ROOTDIR}/devops/docker"
295 MODULES
=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
296 if [ ! -z ${1} ] ; then
297 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
298 for MODULE
in ${POSSIBLE_MODULES}; do
300 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
301 echo "Unknown stage 3 module ${MODULE}"
302 echo "Must be one of ${MODULES}"
306 MODULES
=${POSSIBLE_MODULES}
308 for MODULE
in ${MODULES} ; do
310 docker tag opensourcemano
/${MODULE}:${DEVEL_TAG} ${REGISTRY}/opensourcemano/${MODULE}:${DEVEL_TAG}
311 docker push
${REGISTRY}/opensourcemano/${MODULE}:${DEVEL_TAG}
315 function install_osm
() {
316 cd "${ROOTDIR}/devops/installers"
318 if juju controllers
2>/dev
/null|
grep osm-vca
; then
321 .
/charmed_install.sh
--registry localhost
:32000 --tag ${DEVEL_TAG} ${VCA}
324 function start_robot
() {
325 mkdir
-p "${ROOTDIR}/tests/local"
326 cd "${ROOTDIR}/tests/local"
330 # Workaround for microstack auth URL
331 if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
332 export OS_AUTH_URL
=${OS_AUTH_URL}/v3
335 export OSM_HOSTNAME
=$
(juju config
-m osm nbi site_url |
sed "s/http.*\?:\/\///"):443
336 export PROMETHEUS_HOSTNAME
=$
(juju config
-m osm prometheus site_url |
sed "s/http.*\?:\/\///")
337 export PROMETHEUS_PORT
=80
338 export JUJU_PASSWORD
=`juju gui 2>&1 | grep password | awk '{print $2}'`
339 export HOSTIP
=$
(echo $PROMETHEUS_HOSTNAME |
sed "s/prometheus.//" |
sed "s/.nip.io//")
342 for line
in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg
; done
343 cat << EOF >> robot-systest.cfg
346 ENVIRONMENTS_FOLDER=environments
347 PACKAGES_FOLDER=/robot-systest/osm-packages
353 cat << EOF > robot.etc.hosts
355 ${HOSTIP} prometheus.${HOSTIP}.nip.io nbi.${HOSTIP}.nip.io
357 cat << EOF > clouds.yaml
361 auth_url: $OS_AUTH_URL
362 project_name: $OS_PROJECT_NAME
363 username: $OS_USERNAME
364 password: $OS_PASSWORD
365 user_domain_name: $OS_USER_DOMAIN_NAME
366 project_domain_name: $OS_PROJECT_DOMAIN_NAME
369 VIM_AUTH_URL
=$
(osm vim-show osm |
grep vim_url |
awk '{print $4}' |
tr -d \")
370 if [[ ! -z ${VIM_AUTH_URL} && "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ]] ; then
371 echo "Deleting existing VIM osm as auth URLs have changed"
375 if ! osm vim-show osm
&> /dev
/null
; then
376 echo "Creating VIM osm"
377 if [ -v VIM_VCA
]; then
378 VCA_OPT
="--vca $VIM_VCA"
380 osm vim-create
--name osm
$VCA_OPT --user "$OS_USERNAME" --password "$OS_PASSWORD" \
381 --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
382 --config='{use_floating_ip: True,
383 management_network_name: osm-ext}'
386 if [ ! -z $ROBOT_LOCAL ] ; then
387 LOCAL_MOUNT_1
="/robot-systest/lib"
388 LOCAL_MOUNT_2
="/robot-systest/resources"
389 LOCAL_MOUNT_3
="/robot-systest/testsuite"
391 LOCAL_MOUNT_1
="/tmp/lib"
392 LOCAL_MOUNT_2
="/tmp/resources"
393 LOCAL_MOUNT_3
="/tmp/testsuite"
398 docker run
-ti --entrypoint /bin
/bash \
399 --env OSM_HOSTNAME
=${OSM_HOSTNAME} \
400 --env PROMETHEUS_HOSTNAME
=${PROMETHEUS_HOSTNAME} \
401 --env PROMETHEUS_PORT
=${PROMETHEUS_PORT} \
402 --env JUJU_PASSWORD
=${JUJU_PASSWORD} \
403 --env HOSTIP
=${HOSTIP} \
404 --env-file robot-systest.cfg \
405 -v "$(pwd)/robot.etc.hosts":/etc
/hosts \
406 -v ~
/.osm
/microk8s-config.yaml
:/root
/.kube
/config \
407 -v "$(pwd)/clouds.yaml":/etc
/openstack
/clouds.yaml \
408 -v "${HOME}/snap/qhttp/common"/robot-systest
/reports \
409 -v "${HOME}/snap/qhttp/common:"/robot-systest
/conformance-tests
/reports \
410 -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
411 -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
412 -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
413 opensourcemano
/tests
:$OSM_TESTS_IMAGE_TAG
416 function update_osm_module
() {
417 MODULES
=$
(_find_module_dockerfile
$1)
418 for MODULE
in ${MODULES} ; do
420 echo "Updating ${MODULE}"
421 juju attach-resource
${MODULE} ${MODULE}-image=localhost:32000/opensourcemano/${MODULE}:${DEVEL_TAG}
425 if [ "$0" != "$BASH_SOURCE" ]; then
429 OPTIONS
=$
(show_help |
sed '0,/^OPTIONS:$/d' |
awk '{print $1}')
430 COMPREPLY
=($
(compgen
-W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
433 THIS_SCRIPT
="$(basename ${BASH_SOURCE[0]})"
434 echo "Setting up bash completion for ${THIS_SCRIPT}"
435 complete
-F _osm_local_build
"${THIS_SCRIPT}"
439 eval "${INSTALL_HTTPD}"
440 eval "${INSTALL_LOCAL_REGISTRY}"
441 eval "${INSTALL_MICROSTACK}"
444 eval "${REGISTRY_PUSH}"
445 eval "${INSTALL_OSM}"
446 eval "${UPDATE_INSTALL}"
447 eval "${START_ROBOT}"