3 # Copyright 2020 Telefónica Investigación y Desarrollo S.A.U.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
18 sampling_period
=5 # seconds
19 time_for_readiness
=2 # minutes ready
20 time_for_failure
=7 # minutes broken
21 KUBERNETES
= # By default, assumes Docker Swarm installation
22 STACK_NAME
=osm
# By default, "osm"
24 while getopts "p:r:f:s:k" o
; do
27 sampling_period
=${OPTARG}
30 time_for_readiness
=${OPTARG}
33 time_for_failure
=${OPTARG}
44 oks_threshold
=$
((time_for_readiness
*60/${sampling_period})) # No. ok samples to declare the system ready
45 failures_threshold
=$
((time_for_failure
*60/${sampling_period})) # No. nok samples to declare the system broken
50 ####################################################################################
51 # Loop to check system readiness
52 ####################################################################################
53 while [[ (${failures_in_a_row} -lt ${failures_threshold}) && (${oks_in_a_row} -lt ${oks_threshold}) ]]
56 #------------ CHECKS FOR KUBERNETES INSTALLATION
57 if [ -n "$KUBERNETES" ]
60 # State of Deployments
61 DEPLOYMENTS_STATE
=$
(kubectl get deployment
-n ${STACK_NAME} --no-headers 2>&1)
62 DEPLOYMENTS_READY
=$
(echo "${DEPLOYMENTS_STATE}" |
awk '$2=="1/1" && $4=="1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
63 DEPLOYMENTS_NOT_READY
=$
(echo "${DEPLOYMENTS_STATE}" |
awk '$2!="1/1" || $4!="1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
64 COUNT_DEPLOYMENTS_READY
=$
(echo "${DEPLOYMENTS_READY}"|
grep -v -e '^$' |
wc -l)
65 COUNT_DEPLOYMENTS_NOT_READY
=$
(echo "${DEPLOYMENTS_NOT_READY}" |
grep -v -e '^$' |
wc -l)
67 # State of Statefulsets
68 STS_STATE
=$
(kubectl get statefulset
-n ${STACK_NAME} --no-headers 2>&1)
69 STS_READY
=$
(echo "${STS_STATE}" |
awk '$2=="1/1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
70 STS_NOT_READY
=$
(echo "${STS_STATE}" |
awk '$2!="1/1" {printf ("%20s\t%s\t%s\n", $1, $2, $4)}')
71 COUNT_STS_READY
=$
(echo "${STS_READY}" |
grep -v -e '^$' |
wc -l)
72 COUNT_STS_NOT_READY
=$
(echo "${STS_NOT_READY}" |
grep -v -e '^$' |
wc -l)
75 if [[ $
((${COUNT_DEPLOYMENTS_NOT_READY}+${COUNT_STS_NOT_READY})) -eq 0 ]]
79 echo -ne ===\
> Successful checks
: "${oks_in_a_row}"/${oks_threshold}\\r
82 ((++failures_in_a_row
))
85 echo Bootstraping...
"${failures_in_a_row}" attempts of
${failures_threshold}
87 # Reports failed deployments
88 if [[ "${COUNT_DEPLOYMENTS_NOT_READY}" -ne 0 ]]
90 echo ${COUNT_DEPLOYMENTS_NOT_READY} of $((${COUNT_DEPLOYMENTS_NOT_READY}+${COUNT_DEPLOYMENTS_READY})) deployments starting
:
91 echo "${DEPLOYMENTS_NOT_READY}"
95 # Reports failed statefulsets
96 if [[ "${COUNT_STS_NOT_READY}" -ne 0 ]]
98 echo ${COUNT_STS_NOT_READY} of $((${COUNT_STS_NOT_READY}+${COUNT_STS_READY})) statefulsets starting
:
99 echo "${STS_NOT_READY}"
104 #------------ CHECKS FOR DOCKER SWARM INSTALLATION
106 # State of Docker Services
107 SERVICES_STATE
=$
(sg docker
-c "docker service ls" 2>&1 |
grep " ${STACK_NAME}_")
108 SERVICES_READY
=$
(echo "${SERVICES_STATE}" |
awk '$3=="replicated" && $4=="1/1" {printf ("%20s\t%s\n", $2, $4)}')
109 SERVICES_NOT_READY
=$
(echo "${SERVICES_STATE}" |
awk '$3=="replicated" && $4!="1/1" {printf ("%20s\t%s\n", $2, $4)}')
110 COUNT_SERVICES_READY
=$
(echo "${SERVICES_READY}" |
grep -v -e '^$' |
wc -l)
111 COUNT_SERVICES_NOT_READY
=$
(echo "${SERVICES_NOT_READY}" |
grep -v -e '^$' |
wc -l)
114 if [[ ${COUNT_SERVICES_NOT_READY} -eq 0 ]]
118 echo -ne ===\
> Successful checks
: "${oks_in_a_row}"/${oks_threshold}\\r
121 ((++failures_in_a_row
))
124 echo Bootstraping...
"${failures_in_a_row}" attempts of
${failures_threshold}
125 echo ${COUNT_SERVICES_NOT_READY} of $((${COUNT_SERVICES_NOT_READY}+${COUNT_SERVICES_READY})) services starting
:
126 echo "${SERVICES_NOT_READY}"
130 #------------ NEXT SAMPLE
131 sleep ${sampling_period}
136 ####################################################################################
138 ####################################################################################
139 if [[ (${failures_in_a_row} -ge ${failures_threshold}) ]]
142 echo SYSTEM IS BROKEN