Feature 10967 New option ngsa to install monitoring pipeline architecture
This change covers the installation of Apache Airflow
and Prometheus Pushgateway as an experimental option in
the installer.
Changes are the following:
- `installers/full_install_osm.sh`:
- The installer includes a new option "--ng-sa" to install Airflow and
Prometheus Pushgateway
- When the option is used, the script will call `installers/install_ng_sa.sh`
in turn.
- `installers/install_ng_sa.sh`:
- This script will install Airflow and Pushgateway in the OSM Kubernetes
cluster in osm namespace using the helm charts from the respective communities.
- `installers/helm/values/airflow/values.yaml`:
- File with the values to be used for the installation of Airflow helm chart.
- `docker/Airflow/Dockerfile`:
- Dockerfile used to build the Airflow image, incorporating the DAG Python files,
requirements and internal Python libraries used by DAGs from `osm_ngsa.deb`.
Change-Id: I04cb60b25a9a32e42d4a97fac2d1f6abf868b1f7
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
diff --git a/installers/install_ngsa.sh b/installers/install_ngsa.sh
new file mode 100755
index 0000000..6ddf298
--- /dev/null
+++ b/installers/install_ngsa.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set +eux
+
+# Helm chart 1.6.0 correspondes to Airflow 2.3.0
+AIRFLOW_HELM_VERSION=1.6.0
+PROMPUSHGW_HELM_VERSION=1.18.2
+
+# Install Airflow helm chart
+function install_airflow() {
+ [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
+ # copy airflow-values.yaml to the destination folder
+ sudo mkdir -p ${OSM_HELM_WORK_DIR}
+ sudo cp ${OSM_DEVOPS}/installers/helm/values/airflow-values.yaml ${OSM_HELM_WORK_DIR}
+ if ! helm -n osm status airflow 2> /dev/null ; then
+ # if it does not exist, create secrets and install
+ kubectl -n osm create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
+ helm repo add apache-airflow https://airflow.apache.org
+ helm repo update
+ helm -n osm install airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION}
+ else
+ # if it exists, upgrade
+ helm repo update
+ helm -n osm upgrade airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION}
+ fi
+ [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
+# Install Prometheus Pushgateway helm chart
+function install_prometheus_pushgateway() {
+ [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+ if ! helm -n osm status pushgateway 2> /dev/null ; then
+ # if it does not exist, install
+ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+ helm repo update
+ helm -n osm install pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION}
+ else
+ # if it exists, upgrade
+ helm repo update
+ helm -n osm upgrade pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION}
+ fi
+ [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
+# main
+while getopts ":D:d:-: " o; do
+ case "${o}" in
+ D)
+ OSM_DEVOPS="${OPTARG}"
+ ;;
+ d)
+ OSM_HELM_WORK_DIR="${OPTARG}"
+ ;;
+ -)
+ [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
+ echo -e "Invalid option: '--$OPTARG'\n" >&2
+ exit 1
+ ;;
+ :)
+ echo "Option -$OPTARG requires an argument" >&2
+ exit 1
+ ;;
+ \?)
+ echo -e "Invalid option: '-$OPTARG'\n" >&2
+ exit 1
+ ;;
+ *)
+ exit 1
+ ;;
+ esac
+done
+
+source $OSM_DEVOPS/common/logging
+source $OSM_DEVOPS/common/track
+
+echo "DEBUG_INSTALL=$DEBUG_INSTALL"
+echo "OSM_DEVOPS=$OSM_DEVOPS"
+echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR"
+
+install_airflow
+track deploy_osm airflow_ok
+install_prometheus_pushgateway
+track deploy_osm pushgateway_ok
+