From: Mike Marchetti Date: Thu, 13 Sep 2018 15:38:38 +0000 (-0400) Subject: Add support for multi-stack installation X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=67411aa1754082aeab3860ebb363e61b7e4b72b4;p=osm%2Fdevops.git Add support for multi-stack installation Cleanup use of sudo for copying files. Apply docker specific configuration inside user directory. Utilize multiple juju controllers utilizing the stack name. Create stack with no exposed host ports via '-nohostports' for creating multiple instances of osm (useful in CI) Change-Id: I32e2eab2cbae7fa6939eee2df556f6788d065a92 Signed-off-by: Mike Marchetti --- diff --git a/installers/docker/docker-compose.yaml b/installers/docker/docker-compose.yaml index a1e88bc1..fa279004 100644 --- a/installers/docker/docker-compose.yaml +++ b/installers/docker/docker-compose.yaml @@ -20,7 +20,7 @@ services: kafka: image: wurstmeister/kafka ports: - - "9092:9092" + - "9092" networks: - netOSM environment: @@ -49,7 +49,7 @@ services: OSMNBI_DATABASE_HOST: mongo OSMNBI_MESSAGE_HOST: kafka ports: - - "9999:9999" + - "${OSM_NBI_PORTS:-9999:9999}" #depends_on: # - kafka # - mongo @@ -90,7 +90,7 @@ services: #depends_on: # - ro-db ports: - - "9090:9090" + - "${OSM_RO_PORTS:-9090:9090}" mon: image: osm/mon networks: @@ -105,7 +105,7 @@ services: #depends_on: # - kafka ports: - - "8662:8662" + - "8662" pm: image: osm/pm networks: @@ -124,5 +124,4 @@ services: #depends_on: # - nbi ports: - - "80:80" - + - "${OSM_UI_PORTS:-80:80}" diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index a92e1ba7..60a99e3f 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -26,6 +26,9 @@ function usage(){ echo -e " -b v2.0 (v2.0 branch)" echo -e " -b tags/v1.1.0 (a specific tag)" echo -e " ..." + echo -e " -s user defined stack name, default is osm" + echo -e " -H use specific juju host controller IP" + echo -e " -S use VCA/juju secret key" echo -e " --vimemu: additionally deploy the VIM emulator as a docker container" echo -e " --elk_stack: additionally deploy an ELK docker stack for event logging" echo -e " --pm_stack: additionally deploy a Prometheus+Grafana stack for performance monitoring (PM)" @@ -34,6 +37,9 @@ function usage(){ echo -e " -D use local devops installation path" echo -e " --nolxd: do not install and configure LXD, allowing unattended installations (assumes LXD is already installed and confifured)" echo -e " --nodocker: do not install docker, do not initialize a swarm (assumes docker is already installed and a swarm has been initialized)" + echo -e " --nojuju: do not juju, assumes already installed" + echo -e " --nodockerbuild:do not build docker images (use existing locally cached images)" + echo -e " --nohostports: do not expose docker ports to host (useful for creating multiple instances of osm on the same host)" echo -e " --uninstall: uninstall OSM: remove the containers and delete NAT rules" echo -e " --source: install OSM from source code using the latest stable tag" echo -e " --develop: (deprecated, use '-b master') install OSM from source code using the master branch" @@ -72,6 +78,29 @@ function uninstall(){ return 0 } +# takes a juju/accounts.yaml file and returns the password specific +# for a controller. I wrote this using only bash tools to minimize +# additions of other packages +function parse_juju_password { + password_file="${HOME}/.local/share/juju/accounts.yaml" + local controller_name=$1 + local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + sed -ne "s|^\($s\):|\1|" \ + -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $password_file | + awk -F$fs -v controller=$controller_name '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i /dev/null"; then + # Not found created, create the controller + sg lxd -c "juju bootstrap --bootstrap-series=xenial localhost $OSM_STACK_NAME" + fi + [ $(sg lxd -c "juju controllers" |grep "$OSM_STACK_NAME" |wc -l) -eq 1 ] || FATAL "Juju installation failed" } function generate_docker_images() { @@ -633,34 +668,32 @@ function cmp_overwrite() { file2="$2" if ! $(cmp "${file1}" "${file2}" >/dev/null 2>&1); then if [ -f "${file2}" ]; then - ask_user "The file ${file2} already exists. Overwrite (y/N)? " n && sudo cp -b ${file1} ${file2} + ask_user "The file ${file2} already exists. Overwrite (y/N)? " n && cp -b ${file1} ${file2} else - sudo cp -b ${file1} ${file2} + cp -b ${file1} ${file2} fi fi } function generate_config_log_folders() { echo "Generating config and log folders" - sudo mkdir -p /etc/osm/docker - sudo cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml /etc/osm/docker/docker-compose.yaml - sudo mkdir -p /var/log/osm + cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml echo "Finished generation of config and log folders" } function generate_docker_env_files() { echo "Generating docker env files" - echo "OSMLCM_VCA_HOST=${OSMLCM_VCA_HOST}" |sudo tee /etc/osm/docker/lcm.env - echo "OSMLCM_VCA_SECRET=${OSMLCM_VCA_SECRET}" |sudo tee -a /etc/osm/docker/lcm.env + echo "OSMLCM_VCA_HOST=${OSMLCM_VCA_HOST}" | tee $OSM_DOCKER_WORK_DIR/lcm.env + echo "OSMLCM_VCA_SECRET=${OSMLCM_VCA_SECRET}" | tee -a $OSM_DOCKER_WORK_DIR/lcm.env + MYSQL_ROOT_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` - if [ ! -f /etc/osm/docker/ro-db.env ]; then - echo "MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |sudo tee /etc/osm/docker/ro-db.env + if [ ! -f $OSM_DOCKER_WORK_DIR/ro-db.env ]; then + echo "MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |tee $OSM_DOCKER_WORK_DIR/ro-db.env fi - if [ ! -f /etc/osm/docker/ro.env ]; then - echo "RO_DB_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |sudo tee /etc/osm/docker/ro.env + if [ ! -f $OSM_DOCKER_WORK_DIR/ro.env ]; then + echo "RO_DB_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |tee $OSM_DOCKER_WORK_DIR/ro.env fi - echo "OS_NOTIFIER_URI=http://${DEFAULT_IP}:8662" |tee /tmp/mon.env - cmp_overwrite /tmp/mon.env /etc/osm/docker/mon.env + echo "OS_NOTIFIER_URI=http://${DEFAULT_IP}:8662" |tee $OSM_DOCKER_WORK_DIR/mon.env echo "Finished generation of docker env files" } @@ -678,9 +711,27 @@ function init_docker_swarm() { function deploy_lightweight() { echo "Deploying lightweight build" [ -n "$INSTALL_NODOCKER" ] || init_docker_swarm - remove_stack osm - sg docker -c "docker stack deploy -c /etc/osm/docker/docker-compose.yaml osm" - #docker-compose -f /etc/osm/docker/docker-compose.yaml up -d + remove_stack $OSM_STACK_NAME + + OSM_NBI_PORT=9999 + OSM_RO_PORT=9090 + OSM_UI_PORT=80 + + if [ -n "$NO_HOST_PORTS" ]; then + OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT) + OSM_PORTS+=(OSM_RO_PORTS=$OSM_RO_PORT) + OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT) + else + OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT:$OSM_NBI_PORT) + OSM_PORTS+=(OSM_RO_PORTS=$OSM_RO_PORT:$OSM_RO_PORT) + OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT:$OSM_UI_PORT) + fi + echo "export ${OSM_PORTS[@]}" > $OSM_DOCKER_WORK_DIR/osm_ports.sh + + pushd $OSM_DOCKER_WORK_DIR > /dev/null + sg docker -c "source ./osm_ports.sh; docker stack deploy -c $OSM_DOCKER_WORK_DIR/docker-compose.yaml $OSM_STACK_NAME" + popd > /dev/null + echo "Finished deployment of lightweight build" } @@ -690,11 +741,11 @@ function deploy_elk() { sg docker -c "docker pull docker.elastic.co/logstash/logstash-oss:6.2.3" || FATAL "cannot get logstash docker image" sg docker -c "docker pull docker.elastic.co/kibana/kibana-oss:6.2.3" || FATAL "cannot get kibana docker image" echo "Finished pulling elk docker images" - sudo mkdir -p /etc/osm/docker/osm_elk - sudo cp -b ${OSM_DEVOPS}/installers/docker/osm_elk/* /etc/osm/docker/osm_elk + mkdir -p "$OSM_DOCKER_WORK_DIR/osm_elk" + cp -b ${OSM_DEVOPS}/installers/docker/osm_elk/* $OSM_DOCKER_WORK_DIR/osm_elk remove_stack osm_elk echo "Deploying ELK stack" - sg docker -c "docker stack deploy -c /etc/osm/docker/osm_elk/docker-compose.yml osm_elk" + sg docker -c "docker stack deploy -c $OSM_DOCKER_WORK_DIR/osm_elk/docker-compose.yml osm_elk" echo "Waiting for ELK stack to be up and running" time=0 step=5 @@ -740,17 +791,20 @@ function deploy_perfmon() { echo "Generating osm/kafka-exporter docker image" sg docker -c "docker build ${OSM_DEVOPS}/installers/docker/osm_metrics/kafka-exporter -f ${OSM_DEVOPS}/installers/docker/osm_metrics/kafka-exporter/Dockerfile -t osm/kafka-exporter --no-cache" || FATAL "cannot build kafka-exporter docker image" echo "Finished generation of osm/kafka-exporter docker image" - sudo mkdir -p /etc/osm/docker/osm_metrics - sudo cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.yml /etc/osm/docker/osm_metrics - sudo cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.json /etc/osm/docker/osm_metrics + mkdir -p $OSM_DOCKER_WORK_DIR/osm_metrics + cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.yml $OSM_DOCKER_WORK_DIR/osm_metrics + cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.json $OSM_DOCKER_WORK_DIR/osm_metrics remove_stack osm_metrics echo "Deploying PM stack (Kafka exporter + Prometheus + Grafana)" - sg docker -c "docker stack deploy -c /etc/osm/docker/osm_metrics/docker-compose.yml osm_metrics" + sg docker -c "docker stack deploy -c $OSM_DOCKER_WORK_DIR/osm_metrics/docker-compose.yml osm_metrics" echo "Finished deployment of PM stack" return 0 } function install_lightweight() { + [ -z "$OSM_DOCKER_WORK_DIR" ] && OSM_DOCKER_WORK_DIR="$HOME/.osm/stack/$OSM_STACK_NAME" + [ ! -d "$OSM_DOCKER_WORK_DIR" ] && mkdir -p $OSM_DOCKER_WORK_DIR + [ "$USER" == "root" ] && FATAL "You are running the installer as root. The installer is prepared to be executed as a normal user with sudo privileges." [ -z "$ASSUME_YES" ] && ! ask_user "The installation will configure LXD, install juju, install docker CE and init a docker swarm, as pre-requirements. Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1 track proceed @@ -762,7 +816,9 @@ function install_lightweight() { DEFAULT_IP=`ip -o -4 a |grep ${DEFAULT_IF}|awk '{split($4,a,"/"); print a[1]}'` [ -z "$DEFAULT_IP" ] && FATAL "Not possible to determine the IP address of the interface with the default route" DEFAULT_MTU=$(ip addr show ${DEFAULT_IF} | perl -ne 'if (/mtu\s(\d+)/) {print $1;}') - if [ -z "$INSTALL_NOLXD" ]; then + + # if no host is passed in, we need to install lxd/juju, unless explicilty asked not to + if [ -z "$OSMLCM_VCA_HOST" ] && [ -z "$INSTALL_NOLXD" ]; then need_packages_lw="lxd" echo -e "Checking required packages: $need_packages_lw" dpkg -l $need_packages_lw &>/dev/null \ @@ -775,19 +831,26 @@ function install_lightweight() { || FATAL "failed to install $need_packages_lw" fi track prereqok - install_juju - OSMLCM_VCA_HOST=`sg lxd -c "juju show-controller"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'` - OSMLCM_VCA_SECRET=`grep password ${HOME}/.local/share/juju/accounts.yaml |awk '{print $2}'` - [ -z "$OSMLCM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address" - [ -z "$OSMLCM_VCA_SECRET" ] && FATAL "Cannot obtain juju secret" + [ -z "$INSTALL_NOJUJU" ] && install_juju + + if [ -z "$OSMLCM_VCA_HOST" ]; then + juju_createcontroller + OSMLCM_VCA_HOST=`sg lxd -c "juju show-controller $OSM_STACK_NAME"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'` + [ -z "$OSMLCM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address" + fi + if [ -z "$OSMLCM_VCA_SECRET" ]; then + OSMLCM_VCA_SECRET=$(parse_juju_password $OSM_STACK_NAME) + [ -z "$OSMLCM_VCA_SECRET" ] && FATAL "Cannot obtain juju secret" + fi + track juju [ -n "$INSTALL_NODOCKER" ] || install_docker_ce track docker_ce #install_docker_compose - generate_docker_images + [ -z "$DOCKER_NOBUILD" ] && generate_docker_images track docker_build - generate_config_log_folders generate_docker_env_files + generate_config_log_folders deploy_lightweight track docker_deploy [ -n "$INSTALL_VIMEMU" ] && install_vimemu && track vimemu @@ -809,19 +872,20 @@ function install_vimemu() { git clone https://osm.etsi.org/gerrit/osm/vim-emu.git $EMUTEMPDIR # build vim-emu docker echo "Building vim-emu Docker container..." - sudo docker build -t vim-emu-img -f $EMUTEMPDIR/Dockerfile --no-cache $EMUTEMPDIR/ || FATAL "cannot build vim-emu-img docker image" + + sg docker -c "docker build -t vim-emu-img -f $EMUTEMPDIR/Dockerfile --no-cache $EMUTEMPDIR/" || FATAL "cannot build vim-emu-img docker image" # start vim-emu container as daemon echo "Starting vim-emu Docker container 'vim-emu' ..." if [ -n "$INSTALL_LIGHTWEIGHT" ]; then # in lightweight mode, the emulator needs to be attached to netOSM - sudo docker run --name vim-emu -t -d --restart always --privileged --pid='host' --network=netOSM -v /var/run/docker.sock:/var/run/docker.sock vim-emu-img python examples/osm_default_daemon_topology_2_pop.py + sg docker -c "docker run --name vim-emu -t -d --restart always --privileged --pid='host' --network=netOSM -v /var/run/docker.sock:/var/run/docker.sock vim-emu-img python examples/osm_default_daemon_topology_2_pop.py" else # classic build mode - sudo docker run --name vim-emu -t -d --restart always --privileged --pid='host' -v /var/run/docker.sock:/var/run/docker.sock vim-emu-img python examples/osm_default_daemon_topology_2_pop.py + sg docker -c "docker run --name vim-emu -t -d --restart always --privileged --pid='host' -v /var/run/docker.sock:/var/run/docker.sock vim-emu-img python examples/osm_default_daemon_topology_2_pop.py" fi echo "Waiting for 'vim-emu' container to start ..." sleep 5 - export VIMEMU_HOSTNAME=$(sudo docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' vim-emu) + export VIMEMU_HOSTNAME=$(sg docker -c "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' vim-emu") echo "vim-emu running at ${VIMEMU_HOSTNAME} ..." # print vim-emu connection info echo -e "\nYou might be interested in adding the following vim-emu env variables to your .bashrc file:" @@ -897,12 +961,18 @@ INSTALL_PERFMON="" TO_REBUILD="" INSTALL_NOLXD="" INSTALL_NODOCKER="" +INSTALL_NOJUJU="" NOCONFIGURE="" RELEASE_DAILY="" SESSION_ID=`date +%s` OSM_DEVOPS= +OSMLCM_VCA_HOST= +OSMLCM_VCA_SECRET= +OSM_STACK_NAME=osm +NO_HOST_PORTS="" +DOCKER_NOBUILD="" -while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:" o; do +while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:" o; do case "${o}" in h) usage && exit 0 @@ -931,6 +1001,15 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:" o; do D) OSM_DEVOPS="${OPTARG}" ;; + s) + OSM_STACK_NAME="${OPTARG}" + ;; + H) + OSMLCM_VCA_HOST="${OPTARG}" + ;; + S) + OSMLCM_VCA_SECRET="${OPTARG}" + ;; o) INSTALL_ONLY="y" [ "${OPTARG}" == "vimemu" ] && INSTALL_VIMEMU="y" && continue @@ -968,6 +1047,9 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:" o; do [ "${OPTARG}" == "noconfigure" ] && NOCONFIGURE="y" && continue [ "${OPTARG}" == "showopts" ] && SHOWOPTS="y" && continue [ "${OPTARG}" == "daily" ] && RELEASE_DAILY="y" && continue + [ "${OPTARG}" == "nohostports" ] && NO_HOST_PORTS="y" && continue + [ "${OPTARG}" == "nojuju" ] && INSTALL_NOJUJU="y" && continue + [ "${OPTARG}" == "nodockerbuild" ] && DOCKER_NOBUILD="y" && continue echo -e "Invalid option: '--$OPTARG'\n" >&2 usage && exit 1 ;;