Disable stage3 in branch WIM
[osm/devops.git] / installers / full_install_osm.sh
index cd82909..fbfa12a 100755 (executable)
@@ -105,6 +105,10 @@ function parse_juju_password {
    }'
 }
 
+function generate_secret() {
+    head /dev/urandom | tr -dc A-Za-z0-9 | head -c 32
+}
+
 function remove_volumes() {
     stack=$1
     volumes="mongo_db mon_db osm_packages ro_db"
@@ -384,7 +388,7 @@ function configure_RO(){
 
 function configure_VCA(){
     echo -e "       Configuring VCA"
-    JUJU_PASSWD=`date +%s | sha256sum | base64 | head -c 32`
+    JUJU_PASSWD=$(generate_secret)
     echo -e "$JUJU_PASSWD\n$JUJU_PASSWD" | lxc exec VCA -- juju change-user-password
 }
 
@@ -646,7 +650,7 @@ function generate_docker_images() {
     BUILD_ARGS+=(--build-arg RELEASE="$RELEASE")
     BUILD_ARGS+=(--build-arg REPOSITORY_KEY="$REPOSITORY_KEY")
     BUILD_ARGS+=(--build-arg REPOSITORY_BASE="$REPOSITORY_BASE")
-    
+
     if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q KAFKA ; then
         sg docker -c "docker pull wurstmeister/zookeeper" || FATAL "cannot get zookeeper docker image"
         sg docker -c "docker pull wurstmeister/kafka:${KAFKA_TAG}" || FATAL "cannot get kafka docker image"
@@ -656,22 +660,38 @@ function generate_docker_images() {
         sg docker -c "docker pull mongo" || FATAL "cannot get mongo docker image"
     fi
 
+    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q PROMETHEUS ; then
+        sg docker -c "docker pull prom/prometheus:${PROMETHEUS_TAG}" || FATAL "cannot get prometheus docker image"
+    fi
+
+    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q KEYSTONE-DB ; then
+        sg docker -c "docker pull mariadb:${KEYSTONEDB_TAG}" || FATAL "cannot get keystone-db docker image"
+    fi
+
     if [ -n "$PULL_IMAGES" ]; then
         sg docker -c "docker pull ${DOCKER_USER}/mon:${OSM_DOCKER_TAG}" || FATAL "cannot pull MON docker image"
-        sg docker -c "docker pull ${DOCKER_USER}/pol:${OSM_DOCKER_TAG}" || FATAL "cannot pull POL docker image"
     elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q MON ; then
         git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/MON
         git -C ${LWTEMPDIR}/MON checkout ${COMMIT_ID}
         sg docker -c "docker build ${LWTEMPDIR}/MON -f ${LWTEMPDIR}/MON/docker/Dockerfile -t osm/mon --no-cache" || FATAL "cannot build MON docker image"
-        sg docker -c "docker build ${LWTEMPDIR}/MON/policy_module -f ${LWTEMPDIR}/MON/policy_module/Dockerfile -t osm/pm --no-cache" || FATAL "cannot build PM docker image"
+    fi
+
+    if [ -n "$PULL_IMAGES" ]; then
+        sg docker -c "docker pull ${DOCKER_USER}/pol:${OSM_DOCKER_TAG}" || FATAL "cannot pull POL docker image"
+    elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q MON ; then
+        git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/POL
+        git -C ${LWTEMPDIR}/POL checkout ${COMMIT_ID}
+        sg docker -c "docker build ${LWTEMPDIR}/POL -f ${LWTEMPDIR}/POL/docker/Dockerfile -t osm/pol --no-cache" || FATAL "cannot build PM docker image"
     fi
 
     if [ -n "$PULL_IMAGES" ]; then
         sg docker -c "docker pull ${DOCKER_USER}/nbi:${OSM_DOCKER_TAG}" || FATAL "cannot pull NBI docker image"
+        sg docker -c "docker pull ${DOCKER_USER}/keystone:${OSM_DOCKER_TAG}" || FATAL "cannot pull KEYSTONE docker image"
     elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q NBI ; then
         git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/NBI
         git -C ${LWTEMPDIR}/NBI checkout ${COMMIT_ID}
         sg docker -c "docker build ${LWTEMPDIR}/NBI -f ${LWTEMPDIR}/NBI/Dockerfile.local -t osm/nbi --no-cache" || FATAL "cannot build NBI docker image"
+        sg docker -c "docker build ${LWTEMPDIR}/NBI/keystone -f ${LWTEMPDIR}/NBI/keystone/Dockerfile -t osm/keystone --no-cache" || FATAL "cannot build KEYSTONE docker image"
     fi
 
     if [ -n "$PULL_IMAGES" ]; then
@@ -696,7 +716,7 @@ function generate_docker_images() {
     elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q LW-UI ; then
         git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/LW-UI
         git -C ${LWTEMPDIR}/LW-UI checkout ${COMMIT_ID}
-        sg docker -c "docker build ${LWTEMPDIR}/LW-UI -t osm/light-ui -f ${LWTEMPDIR}/LW-UI/Dockerfile --no-cache" || FATAL "cannot build LW-UI docker image"
+        sg docker -c "docker build ${LWTEMPDIR}/LW-UI -t osm/light-ui -f ${LWTEMPDIR}/LW-UI/docker/Dockerfile --no-cache" || FATAL "cannot build LW-UI docker image"
     fi
 
     if [ -n "$PULL_IMAGES" ]; then
@@ -722,15 +742,31 @@ function cmp_overwrite() {
 function generate_config_log_folders() {
     echo "Generating config and log folders"
     $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml
+    $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus.yml
     echo "Finished generation of config and log folders"
 }
 
 function generate_docker_env_files() {
     echo "Generating docker env files"
-    echo "OSMLCM_VCA_HOST=${OSMLCM_VCA_HOST}" | $WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/lcm.env
-    echo "OSMLCM_VCA_SECRET=${OSMLCM_VCA_SECRET}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    # LCM
+    if [ ! -f $OSM_DOCKER_WORK_DIR/lcm.env ]; then
+        echo "OSMLCM_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    fi
+
+    if ! grep -Fq "OSMLCM_VCA_HOST" $OSM_DOCKER_WORK_DIR/lcm.env; then
+        echo "OSMLCM_VCA_HOST=${OSM_VCA_HOST}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    else
+        $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_HOST.*|OSMLCM_VCA_HOST=$OSM_VCA_HOST|g" $OSM_DOCKER_WORK_DIR/lcm.env
+    fi
 
-    MYSQL_ROOT_PASSWORD=`date +%s | sha256sum | base64 | head -c 32`
+    if ! grep -Fq "OSMLCM_VCA_SECRET" $OSM_DOCKER_WORK_DIR/lcm.env; then
+        echo "OSMLCM_VCA_SECRET=${OSM_VCA_SECRET}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    else
+        $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_SECRET.*|OSMLCM_VCA_SECRET=$OSM_VCA_SECRET|g" $OSM_DOCKER_WORK_DIR/lcm.env
+    fi
+
+    # RO
+    MYSQL_ROOT_PASSWORD=$(generate_secret)
     if [ ! -f $OSM_DOCKER_WORK_DIR/ro-db.env ]; then
         echo "MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/ro-db.env
     fi
@@ -738,31 +774,53 @@ function generate_docker_env_files() {
         echo "RO_DB_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/ro.env
     fi
 
-    MYSQL_ROOT_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` && sleep 1
-    KEYSTONE_DB_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` && sleep 1
-    #ADMIN_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` && sleep 1
-    NBI_PASSWORD=`date +%s | sha256sum | base64 | head -c 32`
+    # Keystone
+    MYSQL_ROOT_PASSWORD=$(generate_secret)
+    KEYSTONE_DB_PASSWORD=$(generate_secret)
+    NBI_PASSWORD=$(generate_secret)
     if [ ! -f $OSM_DOCKER_WORK_DIR/keystone-db.env ]; then
         echo "MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/keystone-db.env
     fi
     if [ ! -f $OSM_DOCKER_WORK_DIR/keystone.env ]; then
         echo "ROOT_DB_PASSWORD=${MYSQL_ROOT_PASSWORD}" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/keystone.env
         echo "KEYSTONE_DB_PASSWORD=${KEYSTONE_DB_PASSWORD}" |$WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/keystone.env
-        #echo "ADMIN_PASSWORD=${ADMIN_PASSWORD}" |$WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/keystone.env
         echo "NBI_PASSWORD=${NBI_PASSWORD}" |$WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/keystone.env
     fi
 
+    # NBI
     if [ ! -f $OSM_DOCKER_WORK_DIR/nbi.env ]; then
         echo "OSMNBI_AUTHENTICATION_SERVICE_PASSWORD=${NBI_PASSWORD}" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/nbi.env
+        echo "OSMNBI_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/nbi.env
     fi
 
-    echo "OS_NOTIFIER_URI=http://${DEFAULT_IP}:8662" |$WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/mon.env
+    # MON
+    if [ ! -f $OSM_DOCKER_WORK_DIR/mon.env ]; then
+        echo "OSMMON_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/mon.env
+    fi
+
+    if ! grep -Fq "OS_NOTIFIER_URI" $OSM_DOCKER_WORK_DIR/mon.env; then
+        echo "OS_NOTIFIER_URI=http://${DEFAULT_IP}:8662" |$WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/mon.env
+    else
+        $WORKDIR_SUDO sed -i "s|OS_NOTIFIER_URI.*|OS_NOTIFIER_URI=http://$DEFAULT_IP:8662|g" $OSM_DOCKER_WORK_DIR/mon.env
+    fi
+
+    if ! grep -Fq "OSMMON_VCA_HOST" $OSM_DOCKER_WORK_DIR/mon.env; then
+        echo "OSMMON_VCA_HOST=${OSM_VCA_HOST}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/mon.env
+    else
+        $WORKDIR_SUDO sed -i "s|OSMMON_VCA_HOST.*|OSMMON_VCA_HOST=$OSM_VCA_HOST|g" $OSM_DOCKER_WORK_DIR/mon.env
+    fi
+
+    if ! grep -Fq "OSMMON_VCA_SECRET" $OSM_DOCKER_WORK_DIR/mon.env; then
+        echo "OSMMON_VCA_SECRET=${OSM_VCA_SECRET}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/mon.env
+    else
+        $WORKDIR_SUDO sed -i "s|OSMMON_VCA_SECRET.*|OSMMON_VCA_SECRET=$OSM_VCA_SECRET|g" $OSM_DOCKER_WORK_DIR/mon.env
+    fi
 
     echo "Finished generation of docker env files"
 }
 
 function generate_osmclient_script () {
-    echo "docker run -ti --network net${OSM_STACK_NAME} osm/osmclient:${OSM_DOCKER_TAG}" | $WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/osm
+    echo "docker run -ti --network net${OSM_STACK_NAME} ${DOCKER_USER}/osmclient:${OSM_DOCKER_TAG}" | $WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/osm
     $WORKDIR_SUDO chmod +x "$OSM_DOCKER_WORK_DIR/osm"
     echo "osmclient sidecar container can be found at: $OSM_DOCKER_WORK_DIR/osm"
 }
@@ -791,6 +849,10 @@ function deploy_lightweight() {
     OSM_KEYSTONE_PORT=5000
     OSM_UI_PORT=80
     OSM_MON_PORT=8662
+    OSM_PROM_PORT=9090
+    OSM_PROM_HOSTPORT=9091
+    [ -n "$INSTALL_ELK" ] && OSM_ELK_PORT=5601
+    [ -n "$INSTALL_PERFMON" ] && OSM_PM_PORT=3000
 
     if [ -n "$NO_HOST_PORTS" ]; then
         OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT)
@@ -798,18 +860,26 @@ function deploy_lightweight() {
         OSM_PORTS+=(OSM_KEYSTONE_PORTS=$OSM_KEYSTONE_PORT)
         OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT)
         OSM_PORTS+=(OSM_MON_PORTS=$OSM_MON_PORT)
+        OSM_PORTS+=(OSM_PROM_PORTS=$OSM_PROM_PORT)
+        [ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT)
+        [ -n "$INSTALL_ELK" ] && OSM_PORTS+=(OSM_ELK_PORTS=$OSM_ELK_PORT)
     else
         OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT:$OSM_NBI_PORT)
         OSM_PORTS+=(OSM_RO_PORTS=$OSM_RO_PORT:$OSM_RO_PORT)
         OSM_PORTS+=(OSM_KEYSTONE_PORTS=$OSM_KEYSTONE_PORT:$OSM_KEYSTONE_PORT)
         OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT:$OSM_UI_PORT)
         OSM_PORTS+=(OSM_MON_PORTS=$OSM_MON_PORT:$OSM_MON_PORT)
+        OSM_PORTS+=(OSM_PROM_PORTS=$OSM_PROM_HOSTPORT:$OSM_PROM_PORT)
+        [ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT:$OSM_PM_PORT)
+        [ -n "$INSTALL_ELK" ] && OSM_PORTS+=(OSM_ELK_PORTS=$OSM_ELK_PORT:$OSM_ELK_PORT)
     fi
     echo "export ${OSM_PORTS[@]}" | $WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export OSM_NETWORK=net${OSM_STACK_NAME}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export TAG=${OSM_DOCKER_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export DOCKER_USER=${DOCKER_USER}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export KAFKA_TAG=${KAFKA_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
+    echo "export PROMETHEUS_TAG=${PROMETHEUS_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
+    echo "export KEYSTONEDB_TAG=${KEYSTONEDB_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
 
 
 
@@ -869,18 +939,14 @@ function deploy_elk() {
 }
 
 function deploy_perfmon() {
-    echo "Pulling docker images for PM (Grafana and Prometheus)"
-    sg docker -c "docker pull prom/prometheus" || FATAL "cannot get prometheus docker image"
+    echo "Pulling docker images for PM (Grafana)"
     sg docker -c "docker pull grafana/grafana" || FATAL "cannot get grafana docker image"
     echo "Finished pulling PM docker images"
-    echo "Generating osm/kafka-exporter docker image"
-    sg docker -c "docker build ${OSM_DEVOPS}/installers/docker/osm_metrics/kafka-exporter -f ${OSM_DEVOPS}/installers/docker/osm_metrics/kafka-exporter/Dockerfile -t osm/kafka-exporter --no-cache" || FATAL "cannot build kafka-exporter docker image"
-    echo "Finished generation of osm/kafka-exporter docker image"
     $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/osm_metrics
     $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.yml $OSM_DOCKER_WORK_DIR/osm_metrics
     $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.json $OSM_DOCKER_WORK_DIR/osm_metrics
     remove_stack osm_metrics
-    echo "Deploying PM stack (Kafka exporter + Prometheus + Grafana)"
+    echo "Deploying PM stack (Grafana)"
     sg docker -c "OSM_NETWORK=net${OSM_STACK_NAME} docker stack deploy -c $OSM_DOCKER_WORK_DIR/osm_metrics/docker-compose.yml osm_metrics"
     echo "Finished deployment of PM stack"
     return 0
@@ -903,8 +969,8 @@ function install_lightweight() {
     DEFAULT_MTU=$(ip addr show ${DEFAULT_IF} | perl -ne 'if (/mtu\s(\d+)/) {print $1;}')
 
     # if no host is passed in, we need to install lxd/juju, unless explicilty asked not to
-    if [ -z "$OSMLCM_VCA_HOST" ] && [ -z "$INSTALL_NOLXD" ]; then
-        need_packages_lw="lxd"
+    if [ -z "$OSM_VCA_HOST" ] && [ -z "$INSTALL_NOLXD" ]; then
+        need_packages_lw="lxd snapd"
         echo -e "Checking required packages: $need_packages_lw"
         dpkg -l $need_packages_lw &>/dev/null \
           || ! echo -e "One or several required packages are not installed. Updating apt cache requires root privileges." \
@@ -918,26 +984,31 @@ function install_lightweight() {
     track prereqok
     [ -z "$INSTALL_NOJUJU" ] && install_juju
 
-    if [ -z "$OSMLCM_VCA_HOST" ]; then
+    if [ -z "$OSM_VCA_HOST" ]; then
         juju_createcontroller
-        OSMLCM_VCA_HOST=`sg lxd -c "juju show-controller $OSM_STACK_NAME"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'`
-        [ -z "$OSMLCM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address"
+        OSM_VCA_HOST=`sg lxd -c "juju show-controller $OSM_STACK_NAME"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'`
+        [ -z "$OSM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address"
     fi
-    if [ -z "$OSMLCM_VCA_SECRET" ]; then
-        OSMLCM_VCA_SECRET=$(parse_juju_password $OSM_STACK_NAME)
-        [ -z "$OSMLCM_VCA_SECRET" ] && FATAL "Cannot obtain juju secret"
+    if [ -z "$OSM_VCA_SECRET" ]; then
+        OSM_VCA_SECRET=$(parse_juju_password $OSM_STACK_NAME)
+        [ -z "$OSM_VCA_SECRET" ] && FATAL "Cannot obtain juju secret"
+    fi
+
+    if [ -z "$OSM_DATABASE_COMMONKEY" ]; then
+        OSM_DATABASE_COMMONKEY=$(generate_secret)
+        [ -z "OSM_DATABASE_COMMONKEY" ] && FATAL "Cannot generate common db secret"
     fi
 
     track juju
     [ -n "$INSTALL_NODOCKER" ] || install_docker_ce
     track docker_ce
     #install_docker_compose
+    [ -n "$INSTALL_NODOCKER" ] || init_docker_swarm
     [ -z "$DOCKER_NOBUILD" ] && generate_docker_images
     track docker_build
     generate_docker_env_files
     generate_config_log_folders
 
-    [ -n "$INSTALL_NODOCKER" ] || init_docker_swarm
     # remove old stack
     remove_stack $OSM_STACK_NAME
     create_docker_network
@@ -1057,8 +1128,8 @@ NOCONFIGURE=""
 RELEASE_DAILY=""
 SESSION_ID=`date +%s`
 OSM_DEVOPS=
-OSMLCM_VCA_HOST=
-OSMLCM_VCA_SECRET=
+OSM_VCA_HOST=
+OSM_VCA_SECRET=
 OSM_STACK_NAME=osm
 NO_HOST_PORTS=""
 DOCKER_NOBUILD=""
@@ -1069,6 +1140,9 @@ OSM_WORK_DIR="/etc/osm"
 OSM_DOCKER_TAG=latest
 DOCKER_USER=osm
 KAFKA_TAG=2.11-1.0.2
+PROMETHEUS_TAG=v2.4.3
+KEYSTONEDB_TAG=10
+OSM_DATABASE_COMMONKEY=
 
 while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:" o; do
     case "${o}" in
@@ -1107,10 +1181,10 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:" o; do
             OSM_STACK_NAME="${OPTARG}"
             ;;
         H)
-            OSMLCM_VCA_HOST="${OPTARG}"
+            OSM_VCA_HOST="${OPTARG}"
             ;;
         S)
-            OSMLCM_VCA_SECRET="${OPTARG}"
+            OSM_VCA_SECRET="${OPTARG}"
             ;;
         w)
             # when specifying workdir, do not use sudo for access