installers/osm_health.sh

   1 #!/bin/sh
   2
   3 # Copyright 2019 ETSI
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 WAIT_TIME=340  # LCM healthcheck needs 2x(30+140) senconds
  19 SERVICES_WITH_HEALTH="nbi ro zookeeper lcm mon pol kafka"
  20 NUM_SERVICES_WITH_HEALTH=$(echo $SERVICES_WITH_HEALTH | wc -w)
  21 WAIT_FINAL=30
  22 OSM_DEPLOYMENT="nbi lcm ro mon pol keystone"
  23 OSM_STATEFULSET="zookeeper kafka mongo mysql prometheus"
  24 NUM_K8S_PODS=$(echo $OSM_DEPLOYMENT $OSM_STATEFULSET | wc -w)
  25
  26 while getopts "w:s:n:c:k" o; do
  27     case "${o}" in
  28         w)
  29             WAIT_TIME=${OPTARG}
  30             ;;
  31         s)
  32             STACK_NAME=${OPTARG}
  33             ;;
  34         n)
  35             NUM_SERVICES_WITH_HEALTH=${OPTARG}
  36             ;;
  37         c)
  38             SERVICES_WITH_HEALTH="${OPTARG}"
  39             ;;
  40         k)
  41             KUBERNETES="y"
  42             ;;
  43     esac
  44 done
  45
  46
  47 time=0
  48 step=2
  49 while [ $time -le "$WAIT_TIME" ]; do
  50     if [ -n "$KUBERNETES" ]; then
  51         if [ "$(kubectl get pods -n "${STACK_NAME}" | grep -i running | wc -l)" -ge "$NUM_K8S_PODS" ]; then
  52             #all pods are running now.
  53             sleep $WAIT_FINAL
  54             exit 0
  55         fi
  56     else
  57         if [ "$(sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | wc -l)" -ge "$NUM_SERVICES_WITH_HEALTH" ]; then
  58             # all dockers are healthy now.
  59             # final sleep is needed until more health checks are added to validate system is ready to handle requests
  60             sleep $WAIT_FINAL
  61             exit 0
  62         fi
  63     fi
  64
  65     sleep $step
  66     time=$((time+step))
  67 done
  68
  69 if [ -n "$KUBERNETES" ]; then
  70     echo "Not all pods are running"
  71     kubectl get pods -n "${STACK_NAME}"
  72     for POD in $OSM_DEPLOYMENT $OSM_STATEFULSET; do
  73         kubectl get pods -n "${STACK_NAME}" | grep -i running | grep -q ^"${POD}-" && continue
  74         echo
  75         echo BEGIN LOGS of pods ${POD} not running
  76         LOG_POD=$(kubectl get pods -n "${STACK_NAME}" | grep -e ^"${POD}-" | awk '{print $1}' )
  77         [ -z "$LOG_POD" ] && echo "${POD} Failed to deploy" || kubectl logs ${LOG_POD} -n $STACK_NAME 2>&1 | tail -n 100
  78         echo END LOGS of services $POD not running
  79     done
  80 else
  81     echo "Not all Docker services are healthy"
  82     sg docker -c "docker ps" | grep " ${STACK_NAME}_"
  83     for S_WITH_HEALTH in $SERVICES_WITH_HEALTH ; do
  84         sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | grep -q "_${S_WITH_HEALTH}."  && continue
  85         echo
  86         echo BEGIN LOGS of container ${S_WITH_HEALTH} not healthy
  87         sg docker -c "docker service logs ${STACK_NAME}_${S_WITH_HEALTH} 2>&1" | tail -n 100
  88         echo END LOGS of container ${S_WITH_HEALTH} not healthy
  89         echo
  90     done
  91 fi
  92
  93 exit 1