installers/k8s/install_osm_k8s_monitoring.sh

   1 #!/bin/bash
   2
   3 #   Copyright 2019 Minsait - Indra S.A.
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #   Author: Jose Manuel Palacios (jmpalacios@minsait.com)
  17 #   Author: Jose Antonio Martinez (jamartinezv@minsait.com)
  18
  19 # Obtain the path where the script is located
  20 HERE=$(dirname $(readlink -f ${BASH_SOURCE[0]}))
  21
  22 echo $HERE
  23 # Load component versions to be deployed
  24 source $HERE/versions_monitoring
  25 V_OPERATOR=""
  26 V_MONGODB_EXPORTER=""
  27 V_MYSQL_EXPORTER=""
  28 V_KAFKA_EXPORTER=""
  29
  30 V_OPERATOR=$PROMETHEUS_OPERATOR
  31 V_MONGODB_EXPORTER=$PROMETHEUS_MONGODB_EXPORTER
  32 V_MYSQL_EXPORTER=$PROMETHEUS_MYSQL_EXPORTER
  33 V_KAFKA_EXPORTER=$PROMETHEUS_KAFKA_EXPORTER
  34
  35
  36 function usage(){
  37     echo -e "usage: $0 [OPTIONS]"
  38     echo -e "Install OSM Monitoring"
  39     echo -e "  OPTIONS"
  40     echo -e "     -n <namespace>   :   namespace to deploy k8s cluster monitoring - default: monitoring"
  41     echo -e "     -o <osm_namespace> : namespace where OSM is installed - default: osm"
  42     echo -e "     -s <service_type>:   service type (ClusterIP|NodePort|LoadBalancer) - default: NodePort"
  43     echo -e "     --debug          :   debug script"
  44     echo -e "     --dump           :   dump arguments and versions"
  45     echo -e "     -h / --help      :   print this help"
  46 }
  47
  48 NAMESPACE=monitoring
  49 OSM_NAMESPACE=osm
  50 HELM=""
  51 DEBUG=""
  52 DUMP_VARS=""
  53 SERVICE_TYPE=""
  54 while getopts ":h-:n:o:s:" o; do
  55     case "${o}" in
  56         h)
  57             usage && exit 0
  58             ;;
  59         n)
  60             NAMESPACE="${OPTARG}"
  61             ;;
  62         o)
  63             OSM_NAMESPACE="${OPTARG}"
  64             ;;
  65         s)
  66             SERVICE_TYPE="${OPTARG}"
  67             ;;
  68         -)
  69             [ "${OPTARG}" == "help" ] && usage && exit 0
  70             [ "${OPTARG}" == "debug" ] && DEBUG="y" && continue
  71             [ "${OPTARG}" == "dump" ] && DUMP_VARS="y" && continue
  72             echo -e "Invalid option: '--$OPTARG'\n" >&2
  73             usage && exit 1
  74             ;;
  75
  76         \?)
  77             echo -e "Invalid option: '-$OPTARG'\n" >&2
  78             usage && exit 1
  79             ;;
  80         *)
  81             usage && exit 1
  82             ;;
  83     esac
  84 done
  85
  86 function dump_vars(){
  87     echo "Args...."
  88     echo "NAMESPACE=$NAMESPACE"
  89     echo "OSM_NAMESPACE=$OSM_NAMESPACE"
  90     echo "SERVICE_TYPE=$SERVICE_TYPE"
  91     echo "DEBUG=$DEBUG"
  92     echo "Versions...."
  93     echo "V_OPERATOR=$V_OPERATOR"
  94     echo "V_MONGODB_EXPORTER=$V_MONGODB_EXPORTER"
  95     echo "V_MYSQL_EXPORTER=$V_MYSQL_EXPORTER"
  96     echo "V_KAFKA_EXPORTER=$V_KAFKA_EXPORTER"
  97 }
  98
  99 # Check K8s version
 100 kubernetes_version=`kubectl version --short | grep Server | awk '{print $3}'`
 101 min_kubernetes_version="v1.16.0"
 102 if [[ "$kubernetes_version" < "$min_kubernetes_version" ]]
 103 then
 104     echo "K8s monitoring could not be installed: Kube-prometheus-stack requires a Kubernetes 1.16+ (current version: $kubernetes_version)"
 105     exit 1
 106 fi
 107
 108 if [ -n "$SERVICE_TYPE" ] ; then
 109     if [ [ $SERVICE_TYPE != "ClusterIP" ] || [ $SERVICE_TYPE != "NodePort" ] || [ $SERVICE_TYPE != "LoadBalancer" ] ] ; then
 110         echo "Wrong service type..."
 111     usage && exit 1
 112     fi
 113 else
 114     SERVICE_TYPE="NodePort"
 115 fi
 116
 117 if [ -n "$DEBUG" ] ; then
 118     set -x
 119 fi
 120
 121 if [ -n "$DUMP_VARS" ] ; then
 122     dump_vars
 123 fi
 124
 125 # Create monitoring namespace
 126 echo "Creating namespace $NAMESPACE"
 127 kubectl create namespace $NAMESPACE
 128
 129 # Needed changes for Kube-Prometheus on Kubeadm installation
 130 # Kube-Controller-Manager
 131 sudo sed -e "s/- --bind-address=127.0.0.1/- --bind-address=0.0.0.0/" -i /etc/kubernetes/manifests/kube-controller-manager.yaml
 132 # Kube-Scheduler
 133 sudo sed -e "s/- --bind-address=127.0.0.1/- --bind-address=0.0.0.0/" -i /etc/kubernetes/manifests/kube-scheduler.yaml
 134 # Kube-Proxy
 135 kubectl -n kube-system get cm/kube-proxy -o yaml > $HERE/kube-proxy-cm.yaml
 136 sed -e "s/metricsBindAddress: \"\"/metricsBindAddress: 0.0.0.0:10249/" -i $HERE/kube-proxy-cm.yaml
 137 kubectl -n kube-system delete cm kube-proxy
 138 kubectl -n kube-system apply -f $HERE/kube-proxy-cm.yaml
 139 rm $HERE/kube-proxy-cm.yaml
 140 kubectl delete pod -l k8s-app=kube-proxy -n kube-system
 141 # Etcd
 142 sudo cp /etc/kubernetes/pki/etcd/healthcheck-client.key $HERE/healthcheck-client.key
 143 sudo chmod a+r $HERE/healthcheck-client.key
 144 kubectl -n $NAMESPACE create secret generic etcd-client-cert --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/healthcheck-client.crt --from-file=$HERE/healthcheck-client.key
 145 sudo awk '/--trusted-ca-file=\/etc\/kubernetes\/pki\/etcd\/ca.crt/ { print; print "    - --metrics=extensive"; next }1' /etc/kubernetes/manifests/etcd.yaml > $HERE/tmp && sudo mv $HERE/tmp /etc/kubernetes/manifests/etcd.yaml
 146 sudo chown root:root  /etc/kubernetes/manifests/etcd.yaml
 147 sudo chmod 600 /etc/kubernetes/manifests/etcd.yaml
 148
 149 # Add Helm prometheus-community repo
 150 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 151 helm repo update
 152
 153 # kube-prometheus-stack installation (previously called prometheus-operator)
 154 $HERE/change-charts-prometheus-operator.sh
 155 echo "Creating stable/kube-prometheus-stack"
 156 cat > $HERE/kube-prometheus-stack-values.yaml <<EOF
 157 kubeControllerManager:
 158   service:
 159     enabled: true
 160     port: 10257
 161     targetPort: 10257
 162   serviceMonitor:
 163     https: true
 164     insecureSkipVerify: true
 165 kubeScheduler:
 166   service:
 167     enabled: true
 168     port: 10259
 169     targetPort: 10259
 170   serviceMonitor:
 171     https: true
 172     insecureSkipVerify: true
 173 kubelet:
 174   serviceMonitor:
 175     https: true
 176 kubeEtcd:
 177   serviceMonitor:
 178    scheme: https
 179    insecureSkipVerify: false
 180    serverName: localhost
 181    caFile: /etc/prometheus/secrets/etcd-client-cert/ca.crt
 182    certFile: /etc/prometheus/secrets/etcd-client-cert/healthcheck-client.crt
 183    keyFile: /etc/prometheus/secrets/etcd-client-cert/healthcheck-client.key
 184 alertmanager:
 185   service:
 186     type: $SERVICE_TYPE
 187 grafana:
 188   enabled: false
 189   forceDeployDashboards: true
 190 prometheus:
 191   service:
 192     type: $SERVICE_TYPE
 193   prometheusSpec:
 194     serviceMonitorSelectorNilUsesHelmValues: false
 195     secrets: ['etcd-client-cert']
 196 EOF
 197 helm install osm-monitoring --namespace $NAMESPACE --version=$V_OPERATOR -f $HERE/kube-prometheus-stack-values.yaml $HERE/helm_charts/kube-prometheus-stack
 198
 199 # Exporters installation
 200 # MongoDB
 201 # exporter
 202 echo "Creating prometheus-community/prometheus-mongodb-exporter"
 203 helm install osm-mongodb-exporter --namespace $NAMESPACE --version=$V_MONGODB_EXPORTER --set serviceMonitor.additionalLabels.release=osm-monitoring,mongodb.uri='mongodb://mongodb-k8s.osm:27017' prometheus-community/prometheus-mongodb-exporter
 204 #dashboard:
 205 kubectl -n $NAMESPACE apply -f $HERE/mongodb-exporter-dashboard.yaml
 206 # Mysql
 207 # exporter
 208 echo "Creating prometheus-community/prometheus-mysql-exporter"
 209 helm install osm-mysql-exporter --namespace $NAMESPACE --version=$V_MYSQL_EXPORTER --set serviceMonitor.enabled=true,serviceMonitor.additionalLabels.release=osm-monitoring,mysql.user="root",mysql.pass=`kubectl -n ${OSM_NAMESPACE} get secret ro-db-secret -o yaml | grep -i -A1 '^data:$' | grep MYSQL_ROOT_PASSWORD | awk '{print $2}' | base64 -d`,mysql.host="mysql.osm",mysql.port="3306",'collectors.info_schema\.tables=true' prometheus-community/prometheus-mysql-exporter
 210 #dashboard:
 211 kubectl -n $NAMESPACE apply -f $HERE/mysql-exporter-dashboard.yaml
 212 # Kafka
 213 # exporter
 214 echo "Creating prometheus-community/prometheus-kafka-exporter"
 215 helm install osm-kafka-exporter --namespace $NAMESPACE --version=$V_KAFKA_EXPORTER --set prometheus.serviceMonitor.enabled=true,prometheus.serviceMonitor.additionalLabels.release=osm-monitoring,kafkaServer={kafka.osm.svc.cluster.local:9092},service.port=9092 prometheus-community/prometheus-kafka-exporter
 216 # dashboard:
 217 kubectl -n $NAMESPACE apply -f $HERE/kafka-exporter-dashboard.yaml
 218
 219 # Deploy summary dashboard
 220 kubectl -n $NAMESPACE apply -f $HERE/summary-dashboard.yaml
 221
 222 # Deploy nodes dashboards
 223 kubectl -n $NAMESPACE apply -f $HERE/nodes-dashboard.yaml
 224