2 #######################################################################################
3 # Copyright ETSI Contributors and Others.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #######################################################################################
20 DIR
="$( cd "$
( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
21 HTTPDDIR
="$( cd "${HOME}/snap
/qhttp
/common
" &> /dev/null && pwd )"
23 KUBECFG
="~/.osm/microk8s-config.yaml"
25 OPENSTACKRC
="/var/snap/microstack/common/etc/microstack.rc"
26 REGISTRY
="localhost:32000"
27 ROOTDIR
="$( cd "${DIR}/..
/..
/" &> /dev/null && pwd)"
28 OSM_TESTS_IMAGE_TAG
="devel"
30 function check_arguments
(){
31 while [ $# -gt 0 ] ; do
34 --apt-proxy) APT_PROXY
="$2" && shift ;;
35 --help |
-h) show_help
&& exit 0 ;;
36 --httpddir) HTTPDIR
="$2" && shift;;
37 --install-local-registry) 'install_local_registry' ;;
38 --install-microstack) 'install_microstack' ;;
39 --install-qhttpd) INSTALL_HTTPD
='install_qhttpd' ;;
40 --kubecfg) KUBECFG
="$2" && shift ;;
41 --module) TARGET_MODULE
="$2" && shift;;
42 --no-cache) NO_CACHE
="--no-cache" ;;
43 --openstackrc) OPENSTACKRC
="$2" && shift ;;
44 --registry) REGISTRY
="$2" && shift;;
45 --robot-local-mounts) ROBOT_LOCAL
=YES
;;
46 --run-tests) TESTS
=YES
;;
47 --vim-vca) VIM_VCA
="$2" && shift;;
48 --osm-tests-image-tag) OSM_TESTS_IMAGE_TAG
="$2" && shift;;
49 stage-2
) STAGE_2
='stage_2 ${TARGET_MODULE}' ;;
50 stage-3
) STAGE_3
='stage_3 ${TARGET_MODULE}' ;;
51 registry-push
) REGISTRY_PUSH
='local_registry_push ${TARGET_MODULE}' ;;
52 install-osm
) INSTALL_OSM
='install_osm' ;;
53 start-robot
) START_ROBOT
='start_robot' ;;
54 update-install
) UPDATE_INSTALL
='update_osm_module ${TARGET_MODULE}'
55 REGISTRY_PUSH
='local_registry_push ${TARGET_MODULE}' ;;
56 *) echo "Unknown option $1"
64 function show_help
() {
67 Perform a local build and potential installation of OSM from sources, using the
68 same process as Jenkins.
71 --help display this help message
72 --apt-proxy provide an apt proxy to docker build steps
73 --debug enable set -x for this script
74 --install-local-registry install and enable Microk8s local registry on port 32000
75 --install-microstack install Microstack and configure to run robot tests
76 --install-qhttpd install QHTTPD as an HTTP server on port ${HTTPPORT}
77 --kubecfg path to kubecfg.yaml (uses Charmed OSM by default)
78 --no-cache do not use any cache when building docker images
79 --module only build this comma delimited list of modules
80 --openstackrc path to Openstack RC file (uses Microstack by default)
81 --registry use this alternate docker registry
82 --run-tests run stage 2 tests
83 --vim-vca name of the a vca registered in OSM to use in the VIM account
84 --osm-tests-image-tag tag to be used in the osm/tests docker image
85 stage-2 run the stage 2 build
86 stage-3 run the stage 3 build
87 registry-push push to the local registry
88 install-osm perform full installation of Charmed OSM from registry
89 start-robot start the Robot test container and leave you at prompt
90 update-install update Charmed OSM with new module container
92 A typical use could be the following:
94 Let's assume that we have different repos cloned in the folder workspace:
97 git clone https://osm.etsi.org/gerrit/osm/devops
98 git clone https://osm.etsi.org/gerrit/osm/NBI
99 git clone https://osm.etsi.org/gerrit/osm/LCM
100 git clone "https://osm.etsi.org/gerrit/osm/RO
101 git clone "https://osm.etsi.org/gerrit/osm/common
102 git clone "https://osm.etsi.org/gerrit/osm/IM
103 git clone "https://osm.etsi.org/gerrit/osm/N2VC
105 First we install a light HTTP server to serve the artifacts:
107 devops/tools/local-build.sh --install-qhttpd
109 Then we generate the artifacts (debian packages) for the different repos: common, IM, N2VC, RO, LCM, NBI
111 devops/tools/local-build.sh --module common,IM,N2VC,RO,LCM,NBI stage-2
113 Then new docker images are generated locally with the tag "devel" (e.g.: opensourcemano/lcm:devel):
115 devops/tools/local-build.sh --module RO,LCM,NBI stage-3
117 Finally, the deployment of OSM will have to be updated to use the new docker images.
122 function print_section
() {
126 function install_local_registry
() {
127 sudo snap
install microk8s
--classic
128 microk8s status
--wait-ready
129 microk8s.
enable registry
132 function install_microstack
() {
133 sudo snap
install microstack
--devmode --edge
134 sudo snap
set microstack config.network.ports.dashboard
=8080
135 sudo microstack.init
--auto --control
137 sudo snap
alias microstack.openstack openstack
138 .
/var
/snap
/microstack
/common
/etc
/microstack.rc
140 for i
in $
(microstack.openstack security group list |
awk '/default/{ print $2 }'); do
141 microstack.openstack security group rule create
$i --protocol icmp
--remote-ip 0.0.0.0/0
142 microstack.openstack security group rule create
$i --protocol tcp
--remote-ip 0.0.0.0/0
145 microstack.openstack network create
--enable --no-share osm-ext
146 microstack.openstack subnet create osm-ext-subnet
--network osm-ext
--dns-nameserver 8.8.8.8 \
147 --subnet-range 172.30.0.0/24
148 microstack.openstack router create external-router
149 microstack.openstack router add subnet external-router osm-ext-subnet
150 microstack.openstack router
set --external-gateway external external-router
152 curl
-L https
://github.com
/cirros-dev
/cirros
/releases
/download
/0.3.5/cirros-0.3
.5-x86_64-disk.img \
153 | microstack.openstack image create
--public --container-format=bare \
154 --disk-format=qcow2 cirros-0.3
.5-x86_64-disk.img
155 curl https
://cloud-images.ubuntu.com
/xenial
/current
/xenial-server-cloudimg-amd64-disk1.img \
156 | microstack.openstack image create
--public --container-format=bare \
157 --disk-format=qcow2 ubuntu16.04
158 curl https
://cloud-images.ubuntu.com
/xenial
/current
/xenial-server-cloudimg-amd64-disk1.img \
159 | microstack.openstack image create
--public --container-format=bare \
160 --disk-format=qcow2 US1604
161 curl https
://cloud-images.ubuntu.com
/bionic
/current
/bionic-server-cloudimg-amd64.img \
162 | microstack.openstack image create
--public --container-format=bare \
163 --disk-format=qcow2 ubuntu18.04
164 curl https
://cloud-images.ubuntu.com
/focal
/current
/focal-server-cloudimg-amd64.img \
165 | microstack.openstack image create
--public --container-format=bare \
166 --disk-format=qcow2 ubuntu20.04
169 function install_qhttpd
() {
170 sudo snap
install qhttp
171 EXISTING_PID
=$
(ps auxw |
grep "http.server $HTTPPORT" |
grep -v grep |
awk '{print $2}')
172 if [ ! -z $EXISTING_PID ] ; then
175 qhttp
-p ${HTTPPORT} &
179 print_section
"Performing Stage 2"
180 MODULES
="common devops IM LCM MON N2VC NBI NG-UI osmclient PLA POL RO tests"
181 if [ ! -z ${1} ] ; then
182 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
183 for MODULE
in ${POSSIBLE_MODULES}; do
184 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
185 echo "Unknown stage 2 module ${MODULE}"
186 echo "Must be one of ${MODULES}"
190 MODULES
=${POSSIBLE_MODULES}
192 print_section
"Cleaning HTTP Directory for full build"
193 rm -fv ${HTTPDDIR}/*.deb
196 for MODULE
in ${MODULES} ; do
198 if [ ! -d ${MODULE} ] ; then
199 echo "Directory ${ROOTDIR}/${MODULE} does not exist"
202 print_section
"Building ${MODULE}"
204 find .
-name '*.deb' -exec rm -v {} \
;
207 if [ ! -z $APT_PROXY ] ; then
208 BUILD_ARGS
="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
210 docker build
${NO_CACHE} ${BUILD_ARGS} -t ${MODULE,,}-stage2 .
212 STAGES
="stage-build.sh"
213 if [ ! -z $TESTS ] ; then
214 STAGES
="stage-test.sh ${STAGES}"
216 for STAGE
in $STAGES ; do
221 bash
-c "groupadd -o -g $(id -g) -r $USER ;
222 useradd -o -u $(id -u) -d /build -r -g $USER $USER ;
223 runuser $USER -c devops-stages/${STAGE}"
224 if [ $?
-ne 0 ] ; then
225 print_section
"Failed to build ${MODULE}"
230 find .
-name '*.deb' -exec mv -v {} ${HTTPDDIR}/ \
;
234 function _find_module_dockerfile
() {
235 cd "${ROOTDIR}/devops/docker"
236 MODULES
=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
237 if [ ! -z ${1} ] ; then
238 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
239 for MODULE
in ${POSSIBLE_MODULES}; do
240 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
241 echo "Unknown stage 3 module ${MODULE}"
242 echo "Must be one of ${MODULES}"
246 echo ${POSSIBLE_MODULES}
253 print_section
"Performing Stage 3"
254 MODULES
=$
(_find_module_dockerfile
$1)
256 if [ ! -z $APT_PROXY ] ; then
257 BUILD_ARGS
="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} "
260 HOSTIP
=$
(ip
-4 addr show docker0 |
grep -Po 'inet \K[\d.]+')
261 for file in ~
/snap
/qhttp
/common
/*.deb
; do
262 file=`basename ${file}`
263 name
=`echo ${file} | cut -d_ -f1 | sed "s/-/_/g"`;
265 BUILD_ARGS
="${BUILD_ARGS}--build-arg ${name}=http://$HOSTIP:${HTTPPORT}/$file "
266 echo Added
${name} as http
://$HOSTIP:${HTTPPORT}/$file
269 for MODULE
in ${MODULES} ; do
270 cd "${ROOTDIR}/devops/docker"
271 if [ ! -d ${MODULE} ] ; then
272 echo "Directory ${ROOTDIR}/${MODULE} does not exist"
275 print_section
"Building ${MODULE}"
278 docker build
${NO_CACHE} -t opensourcemano/${MODULE}:devel ${BUILD_ARGS} .
279 if [ $?
-ne 0 ] ; then
280 print_section
"Failed to build ${MODULE}"
286 function local_registry_push
() {
287 print_section
"Pushing to local registry"
288 cd "${ROOTDIR}/devops/docker"
289 MODULES
=`find . -name Dockerfile -printf '%h\n' |sed 's|\./||' |sort |tr '\n' ' '`
290 if [ ! -z ${1} ] ; then
291 POSSIBLE_MODULES
=$
(echo ${1} |
sed "s/,/ /g")
292 for MODULE
in ${POSSIBLE_MODULES}; do
294 if ! echo "${MODULES}" |
grep -q "${MODULE}" ; then
295 echo "Unknown stage 3 module ${MODULE}"
296 echo "Must be one of ${MODULES}"
300 MODULES
=${POSSIBLE_MODULES}
302 for MODULE
in ${MODULES} ; do
304 docker tag opensourcemano
/${MODULE}:devel ${REGISTRY}/opensourcemano/${MODULE}:devel
305 docker push
${REGISTRY}/opensourcemano
/${MODULE}:devel
309 function install_osm
() {
310 cd "${ROOTDIR}/devops/installers"
312 if juju controllers
2>/dev
/null|
grep osm-vca
; then
315 .
/charmed_install.sh
--registry localhost
:32000 --tag devel
${VCA}
318 function start_robot
() {
319 mkdir
-p "${ROOTDIR}/tests/local"
320 cd "${ROOTDIR}/tests/local"
324 # Workaround for microstack auth URL
325 if [ ${OPENSTACKRC} == "/var/snap/microstack/common/etc/microstack.rc" ] ; then
326 export OS_AUTH_URL
=${OS_AUTH_URL}/v3
329 export OSM_HOSTNAME
=$
(juju config
-m osm nbi site_url |
sed "s/http.*\?:\/\///"):443
330 export PROMETHEUS_HOSTNAME
=$
(juju config
-m osm prometheus site_url |
sed "s/http.*\?:\/\///")
331 export PROMETHEUS_PORT
=80
332 export JUJU_PASSWORD
=`juju gui 2>&1 | grep password | awk '{print $2}'`
333 export HOSTIP
=$
(echo $PROMETHEUS_HOSTNAME |
sed "s/prometheus.//" |
sed "s/.nip.io//")
336 for line
in `env | grep "^OS_" | sort` ; do echo $line >> robot-systest.cfg
; done
337 cat << EOF >> robot-systest.cfg
340 ENVIRONMENTS_FOLDER=environments
341 PACKAGES_FOLDER=/robot-systest/osm-packages
347 cat << EOF > robot.etc.hosts
349 ${HOSTIP} prometheus.${HOSTIP}.nip.io nbi.${HOSTIP}.nip.io
351 cat << EOF > clouds.yaml
355 auth_url: $OS_AUTH_URL
356 project_name: $OS_PROJECT_NAME
357 username: $OS_USERNAME
358 password: $OS_PASSWORD
359 user_domain_name: $OS_USER_DOMAIN_NAME
360 project_domain_name: $OS_PROJECT_DOMAIN_NAME
363 VIM_AUTH_URL
=$
(osm vim-show osm |
grep vim_url |
awk '{print $4}' |
tr -d \")
364 if [[ ! -z ${VIM_AUTH_URL} && "$OS_AUTH_URL" != "${VIM_AUTH_URL}" ]] ; then
365 echo "Deleting existing VIM osm as auth URLs have changed"
369 if ! osm vim-show osm
&> /dev
/null
; then
370 echo "Creating VIM osm"
371 if [ -v VIM_VCA
]; then
372 VCA_OPT
="--vca $VIM_VCA"
374 osm vim-create
--name osm
$VCA_OPT --user "$OS_USERNAME" --password "$OS_PASSWORD" \
375 --auth_url "$OS_AUTH_URL" --tenant "$OS_USERNAME" --account_type openstack \
376 --config='{use_floating_ip: True,
377 management_network_name: osm-ext}'
380 if [ ! -z $ROBOT_LOCAL ] ; then
381 LOCAL_MOUNT_1
="/robot-systest/lib"
382 LOCAL_MOUNT_2
="/robot-systest/resources"
383 LOCAL_MOUNT_3
="/robot-systest/testsuite"
385 LOCAL_MOUNT_1
="/tmp/lib"
386 LOCAL_MOUNT_2
="/tmp/resources"
387 LOCAL_MOUNT_3
="/tmp/testsuite"
392 docker run
-ti --entrypoint /bin
/bash \
393 --env OSM_HOSTNAME
=${OSM_HOSTNAME} \
394 --env PROMETHEUS_HOSTNAME
=${PROMETHEUS_HOSTNAME} \
395 --env PROMETHEUS_PORT
=${PROMETHEUS_PORT} \
396 --env JUJU_PASSWORD
=${JUJU_PASSWORD} \
397 --env HOSTIP
=${HOSTIP} \
398 --env-file robot-systest.cfg \
399 -v "$(pwd)/robot.etc.hosts":/etc
/hosts \
400 -v ~
/.osm
/microk8s-config.yaml
:/root
/.kube
/config \
401 -v "$(pwd)/clouds.yaml":/etc
/openstack
/clouds.yaml \
402 -v "${HOME}/snap/qhttp/common"/robot-systest
/reports \
403 -v "${HOME}/snap/qhttp/common:"/robot-systest
/conformance-tests
/reports \
404 -v "${ROOTDIR}/tests/robot-systest/lib":${LOCAL_MOUNT_1} \
405 -v "${ROOTDIR}/tests/robot-systest/resources":${LOCAL_MOUNT_2} \
406 -v "${ROOTDIR}/tests/robot-systest/testsuite":${LOCAL_MOUNT_3} \
407 opensourcemano
/tests
:$OSM_TESTS_IMAGE_TAG
410 function update_osm_module
() {
411 MODULES
=$
(_find_module_dockerfile
$1)
412 for MODULE
in ${MODULES} ; do
414 echo "Updating ${MODULE}"
415 juju attach-resource
${MODULE} image
=localhost
:32000/opensourcemano
/${MODULE}:devel
419 if [ "$0" != "$BASH_SOURCE" ]; then
423 OPTIONS
=$
(show_help |
sed '0,/^OPTIONS:$/d' |
awk '{print $1}')
424 COMPREPLY
=($
(compgen
-W "${OPTIONS}" -- "${COMP_WORDS[-1]}"))
427 THIS_SCRIPT
="$(basename ${BASH_SOURCE[0]})"
428 echo "Setting up bash completion for ${THIS_SCRIPT}"
429 complete
-F _osm_local_build
"${THIS_SCRIPT}"
433 eval "${INSTALL_HTTPD}"
434 eval "${INSTALL_LOCAL_REGISTRY}"
435 eval "${INSTALL_MICROSTACK}"
438 eval "${REGISTRY_PUSH}"
439 eval "${INSTALL_OSM}"
440 eval "${UPDATE_INSTALL}"
441 eval "${START_ROBOT}"