3 # Copyright 2020 Telefónica Investigación y Desarrollo S.A.U.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
18 sampling_period
=5 # seconds
19 time_for_readiness
=2 # minutes ready
20 time_for_failure
=7 # minutes broken
21 KUBERNETES
= # By default, assumes Docker Swarm installation
22 STACK_NAME
=osm
# By default, "osm"
24 while getopts "p:r:f:s:k" o
; do
27 sampling_period
=${OPTARG}
30 time_for_readiness
=${OPTARG}
33 time_for_failure
=${OPTARG}
44 oks_threshold
=$
((time_for_readiness
*60/${sampling_period})) # No. ok samples to declare the system ready
45 failures_threshold
=$
((time_for_failure
*60/${sampling_period})) # No. nok samples to declare the system broken
50 # Show status of the OSM services deployed with helm
51 echo "helm -n ${STACK_NAME} list"
52 helm
-n ${STACK_NAME} list
53 echo "helm -n ${STACK_NAME} status ${STACK_NAME}"
54 helm
-n ${STACK_NAME} status
${STACK_NAME}
56 ####################################################################################
57 # Loop to check system readiness
58 ####################################################################################
59 while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
62 #------------ CHECKS FOR KUBERNETES INSTALLATION
63 if [ -n "$KUBERNETES" ]
66 # State of Deployments
67 DEPLOYMENTS_STATE
=$
(kubectl get deployment
-n ${STACK_NAME} --no-headers 2>&1)
68 DEPLOYMENTS_READY
=$
(echo "${DEPLOYMENTS_STATE}" |
awk '$2=="1/1" && $4=="1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
69 DEPLOYMENTS_NOT_READY
=$
(echo "${DEPLOYMENTS_STATE}" |
awk '$2!="1/1" || $4!="1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
70 COUNT_DEPLOYMENTS_READY
=$
(echo "${DEPLOYMENTS_READY}"|
grep -v -e '^$' |
wc -l)
71 COUNT_DEPLOYMENTS_NOT_READY
=$
(echo "${DEPLOYMENTS_NOT_READY}" |
grep -v -e '^$' |
wc -l)
73 # State of Statefulsets
74 STS_STATE
=$
(kubectl get statefulset
-n ${STACK_NAME} --no-headers 2>&1)
75 STS_READY
=$
(echo "${STS_STATE}" |
awk '$2=="1/1" || $2=="2/2" || $2=="3/3" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
76 STS_NOT_READY
=$
(echo "${STS_STATE}" |
awk '$2!="1/1" && $2!="2/2" && $2!="3/3" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
77 COUNT_STS_READY
=$
(echo "${STS_READY}" |
grep -v -e '^$' |
wc -l)
78 COUNT_STS_NOT_READY
=$
(echo "${STS_NOT_READY}" |
grep -v -e '^$' |
wc -l)
81 if [[ $
((${COUNT_DEPLOYMENTS_NOT_READY}+${COUNT_STS_NOT_READY})) -eq 0 ]]
85 echo -ne ===\
> Successful checks
: "${oks_in_a_row}"/${oks_threshold}\\r
88 ((++failures_in_a_row
))
91 echo Bootstraping...
"${failures_in_a_row}" attempts of
${failures_threshold}
93 # Reports failed deployments
94 if [[ "${COUNT_DEPLOYMENTS_NOT_READY}" -ne 0 ]]
96 echo ${COUNT_DEPLOYMENTS_NOT_READY} of $((${COUNT_DEPLOYMENTS_NOT_READY}+${COUNT_DEPLOYMENTS_READY})) deployments starting
:
97 echo "${DEPLOYMENTS_NOT_READY}"
101 # Reports failed statefulsets
102 if [[ "${COUNT_STS_NOT_READY}" -ne 0 ]]
104 echo ${COUNT_STS_NOT_READY} of $((${COUNT_STS_NOT_READY}+${COUNT_STS_READY})) statefulsets starting
:
105 echo "${STS_NOT_READY}"
110 #------------ CHECKS FOR DOCKER SWARM INSTALLATION
112 # State of Docker Services
113 SERVICES_STATE
=$
(sg docker
-c "docker service ls" 2>&1 |
grep " ${STACK_NAME}_")
114 SERVICES_READY
=$
(echo "${SERVICES_STATE}" |
awk '$3=="replicated" && $4=="1/1" {printf ("%20s\t%s\n", $2, $4)}')
115 SERVICES_NOT_READY
=$
(echo "${SERVICES_STATE}" |
awk '$3=="replicated" && $4!="1/1" {printf ("%20s\t%s\n", $2, $4)}')
116 COUNT_SERVICES_READY
=$
(echo "${SERVICES_READY}" |
grep -v -e '^$' |
wc -l)
117 COUNT_SERVICES_NOT_READY
=$
(echo "${SERVICES_NOT_READY}" |
grep -v -e '^$' |
wc -l)
120 if [[ ${COUNT_SERVICES_NOT_READY} -eq 0 ]]
124 echo -ne ===\
> Successful checks
: "${oks_in_a_row}"/${oks_threshold}\\r
127 ((++failures_in_a_row
))
130 echo Bootstraping...
"${failures_in_a_row}" attempts of
${failures_threshold}
131 echo ${COUNT_SERVICES_NOT_READY} of $((${COUNT_SERVICES_NOT_READY}+${COUNT_SERVICES_READY})) services starting
:
132 echo "${SERVICES_NOT_READY}"
136 #------------ NEXT SAMPLE
137 sleep ${sampling_period}
142 ####################################################################################
144 ####################################################################################
145 if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
148 echo SYSTEM IS BROKEN