X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=installers%2Fosm_health.sh;h=e7c8065edb68c890cb93589bc318166dacb3ab1e;hb=021b307297a6ab8dfe9ee3af03154989bcd9e0e8;hp=1a9dc7d213396f15ad51260bc39e91f1f0462b56;hpb=0dc45d8a000beccfca12df68336bcfb0bee35bda;p=osm%2Fdevops.git diff --git a/installers/osm_health.sh b/installers/osm_health.sh index 1a9dc7d2..e7c8065e 100755 --- a/installers/osm_health.sh +++ b/installers/osm_health.sh @@ -1,11 +1,29 @@ #!/bin/sh +# Copyright 2019 ETSI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + WAIT_TIME=340 # LCM healthcheck needs 2x(30+140) senconds -SERVICES_WITH_HEALTH="nbi ro zookeeper lcm mon" +SERVICES_WITH_HEALTH="nbi ro zookeeper lcm mon light-ui pol kafka" NUM_SERVICES_WITH_HEALTH=$(echo $SERVICES_WITH_HEALTH | wc -w) WAIT_FINAL=30 +OSM_DEPLOYMENT="light-ui nbi lcm ro mon pol keystone" +OSM_STATEFULSET="zookeeper kafka mongo mysql prometheus" +NUM_K8S_PODS=$(echo $OSM_DEPLOYMENT $OSM_STATEFULSET | wc -w) -while getopts "w:s:n:c:" o; do +while getopts "w:s:n:c:k" o; do case "${o}" in w) WAIT_TIME=${OPTARG} @@ -19,6 +37,9 @@ while getopts "w:s:n:c:" o; do c) SERVICES_WITH_HEALTH="${OPTARG}" ;; + k) + KUBERNETES="y" + ;; esac done @@ -26,28 +47,47 @@ done time=0 step=2 while [ $time -le "$WAIT_TIME" ]; do - if [ "$(docker ps | grep " ${STACK_NAME}_" | grep -i healthy | wc -l)" -ge "$NUM_SERVICES_WITH_HEALTH" ]; then - # all dockers are healthy now. - # final sleep is needed until more health checks are added to validate system is ready to handle requests - sleep $WAIT_FINAL - exit 0 + if [ -n "$KUBERNETES" ]; then + if [ "$(kubectl get pods -n "${STACK_NAME}" | grep -i running | wc -l)" -ge "$NUM_K8S_PODS" ]; then + #all pods are running now. + sleep $WAIT_FINAL + exit 0 + fi + else + if [ "$(sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | wc -l)" -ge "$NUM_SERVICES_WITH_HEALTH" ]; then + # all dockers are healthy now. + # final sleep is needed until more health checks are added to validate system is ready to handle requests + sleep $WAIT_FINAL + exit 0 + fi fi sleep $step time=$((time+step)) done -echo "Not all Docker services are healthy" -docker ps | grep " ${STACK_NAME}_" - -for S_WITH_HEALTH in $SERVICES_WITH_HEALTH ; do - docker ps | grep " ${STACK_NAME}_" | grep -i healthy | grep -q "_${S_WITH_HEALTH}." && continue - echo - echo BEGIN LOGS of container ${S_WITH_HEALTH} not healthy - docker service logs ${STACK_NAME}_${S_WITH_HEALTH} 2>&1 | tail -n 100 - echo END LOGS of container ${S_WITH_HEALTH} not healthy - echo -done +if [ -n "$KUBERNETES" ]; then + echo "Not all pods are running" + kubectl get pods -n "${STACK_NAME}" + for POD in $OSM_DEPLOYMENT $OSM_STATEFULSET; do + kubectl get pods -n "${STACK_NAME}" | grep -i running | grep -q ^"${POD}-" && continue + echo + echo BEGIN LOGS of pods ${POD} not running + LOG_POD=$(kubectl get pods -n "${STACK_NAME}" | grep -e ^"${POD}-" | awk '{print $1}' ) + [ -z "$LOG_POD" ] && echo "${POD} Failed to deploy" || kubectl logs ${LOG_POD} -n $STACK_NAME 2>&1 | tail -n 100 + echo END LOGS of services $POD not running + done +else + echo "Not all Docker services are healthy" + sg docker -c "docker ps" | grep " ${STACK_NAME}_" + for S_WITH_HEALTH in $SERVICES_WITH_HEALTH ; do + sg docker -c "docker ps" | grep " ${STACK_NAME}_" | grep -i healthy | grep -q "_${S_WITH_HEALTH}." && continue + echo + echo BEGIN LOGS of container ${S_WITH_HEALTH} not healthy + sg docker -c "docker service logs ${STACK_NAME}_${S_WITH_HEALTH} 2>&1" | tail -n 100 + echo END LOGS of container ${S_WITH_HEALTH} not healthy + echo + done +fi exit 1 -