installers/osm_health.sh

   1 #!/bin/sh
   2
   3 # Copyright 2019 ETSI
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  14 # implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 WAIT_TIME=340  # LCM healthcheck needs 2x(30+140) senconds
  19 #SERVICES_WITH_HEALTH="nbi ro zookeeper lcm mon light-ui pol kafka"
  20 SERVICES_WITH_HEALTH="nbi ro zookeeper lcm mon pol kafka"
  21 NUM_SERVICES_WITH_HEALTH=$(echo $SERVICES_WITH_HEALTH | wc -w)
  22 WAIT_FINAL=30
  23 #OSM_DEPLOYMENT="light-ui nbi lcm ro mon pol keystone"
  24 OSM_DEPLOYMENT="nbi lcm ro mon pol keystone"
  25 OSM_STATEFULSET="zookeeper kafka mongo mysql prometheus"
  26 NUM_K8S_PODS=$(echo $OSM_DEPLOYMENT $OSM_STATEFULSET | wc -w)
  27
  28 while getopts "w:s:n:c:k" o; do
  29     case "${o}" in
  30         w)
  31             WAIT_TIME=${OPTARG}
  32             ;;
  33         s)
  34             STACK_NAME=${OPTARG}
  35             ;;
  36         n)
  37             NUM_SERVICES_WITH_HEALTH=${OPTARG}
  38             ;;
  39         c)
  40             SERVICES_WITH_HEALTH="${OPTARG}"
  41             ;;
  42         k)
  43             KUBERNETES="y"
  44             ;;
  45     esac
  46 done
  47
  48
  49 time=0
  50 step=2
  51 while [ $time -le "$WAIT_TIME" ]; do
  52     if [ -n "$KUBERNETES" ]; then
  53         if [ "$(kubectl get pods -n "${STACK_NAME}" | grep -i running | wc -l)" -ge "$NUM_K8S_PODS" ]; then
  54             #all pods are running now.
  55             sleep $WAIT_FINAL
  56             exit 0
  57         fi
  58     else
  59         if [ "$(sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | wc -l)" -ge "$NUM_SERVICES_WITH_HEALTH" ]; then
  60             # all dockers are healthy now.
  61             # final sleep is needed until more health checks are added to validate system is ready to handle requests
  62             sleep $WAIT_FINAL
  63             exit 0
  64         fi
  65     fi
  66
  67     sleep $step
  68     time=$((time+step))
  69 done
  70
  71 if [ -n "$KUBERNETES" ]; then
  72     echo "Not all pods are running"
  73     kubectl get pods -n "${STACK_NAME}"
  74     for POD in $OSM_DEPLOYMENT $OSM_STATEFULSET; do
  75         kubectl get pods -n "${STACK_NAME}" | grep -i running | grep -q ^"${POD}-" && continue
  76         echo
  77         echo BEGIN LOGS of pods ${POD} not running
  78         LOG_POD=$(kubectl get pods -n "${STACK_NAME}" | grep -e ^"${POD}-" | awk '{print $1}' )
  79         [ -z "$LOG_POD" ] && echo "${POD} Failed to deploy" || kubectl logs ${LOG_POD} -n $STACK_NAME 2>&1 | tail -n 100
  80         echo END LOGS of services $POD not running
  81     done
  82 else
  83     echo "Not all Docker services are healthy"
  84     sg docker -c "docker ps" | grep " ${STACK_NAME}_"
  85     for S_WITH_HEALTH in $SERVICES_WITH_HEALTH ; do
  86         sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | grep -q "_${S_WITH_HEALTH}."  && continue
  87         echo
  88         echo BEGIN LOGS of container ${S_WITH_HEALTH} not healthy
  89         sg docker -c "docker service logs ${STACK_NAME}_${S_WITH_HEALTH} 2>&1" | tail -n 100
  90         echo END LOGS of container ${S_WITH_HEALTH} not healthy
  91         echo
  92     done
  93 fi
  94
  95 exit 1