From: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Fri, 4 Nov 2022 22:18:48 +0000 (+0100)
Subject: Feature 10967 New option ngsa to install monitoring pipeline architecture
X-Git-Tag: release-v13.0-start~13
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=7b53d2629d0c181e026e4eaa1b271b12f36c85f5;p=osm%2Fdevops.git

Feature 10967 New option ngsa to install monitoring pipeline architecture

This change covers the installation of Apache Airflow
and Prometheus Pushgateway as an experimental option in
the installer.

Changes are the following:
- `installers/full_install_osm.sh`:
  - The installer includes a new option "--ng-sa" to install Airflow and
    Prometheus Pushgateway
  - When the option is used, the script will call `installers/install_ng_sa.sh`
    in turn.
- `installers/install_ng_sa.sh`:
  - This script will install Airflow and Pushgateway in the OSM Kubernetes
    cluster in osm namespace using the helm charts from the respective communities.
- `installers/helm/values/airflow/values.yaml`:
   - File with the values to be used for the installation of Airflow helm chart.
- `docker/Airflow/Dockerfile`:
  - Dockerfile used to build the Airflow image, incorporating the DAG Python files,
    requirements and internal Python libraries used by DAGs from `osm_ngsa.deb`.

Change-Id: I04cb60b25a9a32e42d4a97fac2d1f6abf868b1f7
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
---

diff --git a/docker/Airflow/Dockerfile b/docker/Airflow/Dockerfile
new file mode 100644
index 00000000..b350a09b
--- /dev/null
+++ b/docker/Airflow/Dockerfile
@@ -0,0 +1,37 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+FROM apache/airflow:2.3.0-python3.8
+USER root
+RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
+    DEBIAN_FRONTEND=noninteractive apt-get --yes install \
+    gcc git python3 python3-dev python3-venv python3-pip \
+    python3-setuptools wget && \
+    python3 -m pip install -U pip build
+
+ARG PYTHON3_OSM_COMMON_URL
+ARG PYTHON3_OSM_NGSA_URL
+RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb
+RUN dpkg -i ./osm_common.deb
+# RUN curl $PYTHON3_OSM_NGSA_URL -o osm_ngsa.deb
+# RUN dpkg -i ./osm_ngsa.deb
+
+RUN pip3 install \
+    -r /usr/lib/python3/dist-packages/osm_common/requirements.txt
+#    -r /usr/lib/python3/dist-packages/osm_ngsa/requirements.txt
+
+USER airflow
diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh
index 84249a5d..50b2c031 100755
--- a/installers/full_install_osm.sh
+++ b/installers/full_install_osm.sh
@@ -36,6 +36,7 @@ function usage(){
     echo -e "     -P <VCA pubkey> use VCA/juju public key file"
     echo -e "     -A <VCA apiproxy> use VCA/juju API proxy"
     echo -e "     --pla:          install the PLA module for placement support"
+    echo -e "     --ng-sa:        install Airflow and Pushgateway to get VNF and NS status (experimental)"
     echo -e "     -m <MODULE>:    install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)"
     echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (k8s_monitor)"
     echo -e "     -O <openrc file path/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
@@ -310,27 +311,6 @@ function generate_k8s_manifest_files() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
     #Kubernetes resources
     sudo cp -bR ${OSM_DEVOPS}/installers/docker/osm_pods $OSM_DOCKER_WORK_DIR
-    sudo rm -f ${OSM_DOCKER_WORK_DIR}/osm_pods/ng-prometheus.yaml
-    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
-}
-
-function generate_prometheus_grafana_files() {
-    #this only works with docker swarm
-    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    # Prometheus files
-    sudo mkdir -p $OSM_DOCKER_WORK_DIR/prometheus
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/prometheus/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus/prometheus.yml
-
-    # Grafana files
-    sudo mkdir -p $OSM_DOCKER_WORK_DIR/grafana
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/dashboards-osm.yml $OSM_DOCKER_WORK_DIR/grafana/dashboards-osm.yml
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/datasource-prometheus.yml $OSM_DOCKER_WORK_DIR/grafana/datasource-prometheus.yml
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-sample-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-sample-dashboard.json
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-system-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-system-dashboard.json
-
-    # Prometheus Exporters files
-    sudo mkdir -p $OSM_DOCKER_WORK_DIR/prometheus_exporters
-    sudo cp -b ${OSM_DEVOPS}/installers/docker/prometheus_exporters/node_exporter.service $OSM_DOCKER_WORK_DIR/prometheus_exporters/node_exporter.service
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
@@ -345,6 +325,9 @@ function generate_docker_env_files() {
     sudo cp $OSM_DOCKER_WORK_DIR/pol.env{,~}
     sudo cp $OSM_DOCKER_WORK_DIR/ro-db.env{,~}
     sudo cp $OSM_DOCKER_WORK_DIR/ro.env{,~}
+    if [ -n "${INSTALL_NGSA}" ]; then
+        sudo cp $OSM_DOCKER_WORK_DIR/ngsa.env{,~}
+    fi
 
     echo "Generating docker env files"
     # LCM
@@ -465,12 +448,16 @@ function generate_docker_env_files() {
         sudo sed -i "s|OSMMON_VCA_CACERT.*|OSMMON_VCA_CACERT=${OSM_VCA_CACERT}|g" $OSM_DOCKER_WORK_DIR/mon.env
     fi
 
-
     # POL
     if [ ! -f $OSM_DOCKER_WORK_DIR/pol.env ]; then
         echo "OSMPOL_SQL_DATABASE_URI=mysql://root:${MYSQL_ROOT_PASSWORD}@mysql:3306/pol" | sudo tee -a $OSM_DOCKER_WORK_DIR/pol.env
     fi
 
+    # NG-SA
+    if [ -n "${INSTALL_NGSA}" ] && [ ! -f $OSM_DOCKER_WORK_DIR/ngsa.env ]; then
+        echo "OSMMON_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | sudo tee -a $OSM_DOCKER_WORK_DIR/ngsa.env
+    fi
+
     echo "Finished generation of docker env files"
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
@@ -486,6 +473,9 @@ function kube_secrets(){
     kubectl create secret generic ro-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ro.env
     kubectl create secret generic keystone-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/keystone.env
     kubectl create secret generic pol-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env
+    if [ -n "${INSTALL_NGSA}" ]; then
+        kubectl create secret generic ngsa-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ngsa.env
+    fi
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
@@ -506,13 +496,18 @@ function deploy_charmed_services() {
 
 function deploy_osm_pla_service() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    # corresponding to namespace_vol
-    sudo  sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_DOCKER_WORK_DIR/osm_pla/pla.yaml
     # corresponding to deploy_osm_services
     kubectl apply -n $OSM_STACK_NAME -f $OSM_DOCKER_WORK_DIR/osm_pla
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
+function install_osm_ngsa_service() {
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
+    $OSM_DEVOPS/installers/install_ngsa.sh -d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \
+    FATAL_TRACK install_osm_ngsa_service "install_ngsa.sh failed"
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
 function parse_yaml() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
     TAG=$1
@@ -521,19 +516,23 @@ function parse_yaml() {
     for module in $services; do
         if [ "$module" == "pla" ]; then
             if [ -n "$INSTALL_PLA" ]; then
-                echo "Updating K8s manifest file from opensourcemano\/${module}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${module}:${TAG}"
+                echo "Updating K8s manifest file from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}"
                 sudo sed -i "s#opensourcemano/pla:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/pla:${TAG}#g" ${OSM_DOCKER_WORK_DIR}/osm_pla/pla.yaml
             fi
         else
-            echo "Updating K8s manifest file from opensourcemano\/${module}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${module}:${TAG}"
-            sudo sed -i "s#opensourcemano/${module}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml
+            image=${module}
+            if [ "$module" == "ng-prometheus" ]; then
+                image="prometheus"
+            fi
+            echo "Updating K8s manifest file from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}"
+            sudo sed -i "s#opensourcemano/${image}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${image}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml
         fi
     done
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
 function update_manifest_files() {
-    osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus"
+    osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus ng-prometheus"
     list_of_services=""
     for module in $osm_services; do
         module_upper="${module^^}"
@@ -547,14 +546,23 @@ function update_manifest_files() {
     if [ -n "$MODULE_DOCKER_TAG" ]; then
         parse_yaml $MODULE_DOCKER_TAG $list_of_services_to_rebuild
     fi
+    # The manifest for prometheus is prometheus.yaml or ng-prometheus.yaml, depending on the installation option
+    if [ -n "$INSTALL_NGSA" ]; then
+        sudo rm -f ${OSM_K8S_WORK_DIR}/prometheus.yaml
+    else
+        sudo rm -f ${OSM_K8S_WORK_DIR}/ng-prometheus.yaml
+    fi
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
 function namespace_vol() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    osm_services="nbi lcm ro pol mon kafka mysql prometheus"
+    # List of services with a volume mounted in path /var/lib/osm
+    osm_services="mysql"
     for osm in $osm_services; do
-        sudo  sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_K8S_WORK_DIR/$osm.yaml
+        if [ -f "$OSM_K8S_WORK_DIR/$osm.yaml" ] ; then
+            sudo sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_K8S_WORK_DIR/$osm.yaml
+        fi
     done
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
@@ -685,7 +693,7 @@ function install_osm() {
     track deploy_osm namespace_vol_ok
     deploy_osm_services
     track deploy_osm deploy_osm_services_k8s_ok
-    if [ -n "$INSTALL_PLA"]; then
+    if [ -n "$INSTALL_PLA" ]; then
         # optional PLA install
         deploy_osm_pla_service
         track deploy_osm deploy_osm_pla_ok
@@ -695,6 +703,11 @@ function install_osm() {
         install_k8s_monitoring
         track deploy_osm install_k8s_monitoring_ok
     fi
+    if [ -n "$INSTALL_NGSA" ]; then
+        # optional PLA install
+        install_osm_ngsa_service
+        track deploy_osm install_osm_ngsa_ok
+    fi
 
     [ -z "$INSTALL_NOHOSTCLIENT" ] && install_osmclient
     track osmclient osmclient_ok
@@ -792,6 +805,7 @@ function dump_vars(){
     echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR"
     echo "INSTALL_LIGHTWEIGHT=$INSTALL_LIGHTWEIGHT"
     echo "INSTALL_LXD=$INSTALL_LXD"
+    echo "INSTALL_NGSA=$INSTALL_NGSA"
     echo "INSTALL_NODOCKER=$INSTALL_NODOCKER"
     echo "INSTALL_NOJUJU=$INSTALL_NOJUJU"
     echo "INSTALL_NOLXD=$INSTALL_NOLXD"
@@ -809,6 +823,7 @@ function dump_vars(){
     echo "OSM_DEVOPS=$OSM_DEVOPS"
     echo "OSM_DOCKER_TAG=$OSM_DOCKER_TAG"
     echo "OSM_DOCKER_WORK_DIR=$OSM_DOCKER_WORK_DIR"
+    echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR"
     echo "OSM_K8S_WORK_DIR=$OSM_K8S_WORK_DIR"
     echo "OSM_STACK_NAME=$OSM_STACK_NAME"
     echo "OSM_VCA_HOST=$OSM_VCA_HOST"
@@ -861,6 +876,7 @@ RELEASE="ReleaseTEN"
 REPOSITORY="stable"
 INSTALL_VIMEMU=""
 INSTALL_PLA=""
+INSTALL_NGSA=""
 LXD_REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/lxd"
 LXD_REPOSITORY_PATH=""
 INSTALL_LIGHTWEIGHT="y"
@@ -892,8 +908,9 @@ DOCKER_NOBUILD=""
 REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg"
 REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/debian"
 OSM_WORK_DIR="/etc/osm"
-OSM_DOCKER_WORK_DIR="/etc/osm/docker"
+OSM_DOCKER_WORK_DIR="${OSM_WORK_DIR}/docker"
 OSM_K8S_WORK_DIR="${OSM_DOCKER_WORK_DIR}/osm_pods"
+OSM_HELM_WORK_DIR="${OSM_WORK_DIR}/helm"
 OSM_HOST_VOL="/var/lib/osm"
 OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}"
 OSM_DOCKER_TAG=latest
@@ -1057,6 +1074,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o;
             [ "${OPTARG}" == "tag" ] && continue
             [ "${OPTARG}" == "registry" ] && continue
             [ "${OPTARG}" == "pla" ] && INSTALL_PLA="y" && continue
+            [ "${OPTARG}" == "ng-sa" ] && INSTALL_NGSA="y" && continue
             [ "${OPTARG}" == "volume" ] && OPENSTACK_ATTACH_VOLUME="true" && continue
             [ "${OPTARG}" == "nocachelxdimages" ] && continue
             [ "${OPTARG}" == "cachelxdimages" ] && INSTALL_CACHELXDIMAGES="--cachelxdimages" && continue
diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml
new file mode 100644
index 00000000..0a370972
--- /dev/null
+++ b/installers/helm/values/airflow-values.yaml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+defaultAirflowRepository: gerardogarcia/airflow
+defaultAirflowTag: "0.3"
+webserverSecretKeySecretName: airflow-webserver-secret
+webserver:
+  service:
+    type: NodePort
+    ports:
+      - name: airflow-ui
+        port: "{{ .Values.ports.airflowUI }}"
+        targetPort: "{{ .Values.ports.airflowUI }}"
+extraEnvFrom: |
+  - secretRef:
+      name: ngsa-secret
+
diff --git a/installers/install_ngsa.sh b/installers/install_ngsa.sh
new file mode 100755
index 00000000..6ddf2986
--- /dev/null
+++ b/installers/install_ngsa.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+set +eux
+
+# Helm chart 1.6.0 correspondes to Airflow 2.3.0
+AIRFLOW_HELM_VERSION=1.6.0
+PROMPUSHGW_HELM_VERSION=1.18.2
+
+# Install Airflow helm chart
+function install_airflow() {
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
+    # copy airflow-values.yaml to the destination folder
+    sudo mkdir -p ${OSM_HELM_WORK_DIR}
+    sudo cp ${OSM_DEVOPS}/installers/helm/values/airflow-values.yaml ${OSM_HELM_WORK_DIR}
+    if ! helm -n osm status airflow 2> /dev/null ; then
+        # if it does not exist, create secrets and install
+        kubectl -n osm create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
+        helm repo add apache-airflow https://airflow.apache.org
+        helm repo update
+        helm -n osm install airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION}
+    else
+        # if it exists, upgrade
+        helm repo update
+        helm -n osm upgrade airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION}
+    fi
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
+# Install Prometheus Pushgateway helm chart
+function install_prometheus_pushgateway() {
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+    if ! helm -n osm status pushgateway 2> /dev/null ; then
+        # if it does not exist, install
+        helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+        helm repo update
+        helm -n osm install pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION}
+    else
+        # if it exists, upgrade
+        helm repo update
+        helm -n osm upgrade pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION}
+    fi
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
+# main
+while getopts ":D:d:-: " o; do
+    case "${o}" in
+        D)
+            OSM_DEVOPS="${OPTARG}"
+            ;;
+        d)
+            OSM_HELM_WORK_DIR="${OPTARG}"
+            ;;
+        -)
+            [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
+            echo -e "Invalid option: '--$OPTARG'\n" >&2
+            exit 1
+            ;;
+        :)
+            echo "Option -$OPTARG requires an argument" >&2
+            exit 1
+            ;;
+        \?)
+            echo -e "Invalid option: '-$OPTARG'\n" >&2
+            exit 1
+            ;;
+        *)
+            exit 1
+            ;;
+    esac
+done
+
+source $OSM_DEVOPS/common/logging
+source $OSM_DEVOPS/common/track
+
+echo "DEBUG_INSTALL=$DEBUG_INSTALL"
+echo "OSM_DEVOPS=$OSM_DEVOPS"
+echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR"
+
+install_airflow
+track deploy_osm airflow_ok
+install_prometheus_pushgateway
+track deploy_osm pushgateway_ok
+
diff --git a/installers/install_osm.sh b/installers/install_osm.sh
index 7199c3b7..11ce0bec 100755
--- a/installers/install_osm.sh
+++ b/installers/install_osm.sh
@@ -38,15 +38,12 @@ function usage(){
     echo -e "     -H <VCA host>   use specific juju host controller IP"
     echo -e "     -S <VCA secret> use VCA/juju secret key"
     echo -e "     -P <VCA pubkey> use VCA/juju public key file"
-    echo -e "     -C <VCA cacert> use VCA/juju CA certificate file"
     echo -e "     -A <VCA apiproxy> use VCA/juju API proxy"
     echo -e "     --pla:          install the PLA module for placement support"
+    echo -e "     --ng-sa:        install Airflow and Pushgateway to get VNF and NS status (experimental)"
     echo -e "     -m <MODULE>:    install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)"
     echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (k8s_monitor)"
     echo -e "     -O <openrc file path/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
-    echo -e "     -m <MODULE>:    install OSM but only rebuild the specified docker images (LW-UI, NBI, LCM, RO, MON, POL, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, PLA, NONE)"
-    echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (vimemu, elk_stack, k8s_monitor)"
-    echo -e "     -O <openrc file/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
     echo -e "     -N <openstack public network name/ID>: Public network name required to setup OSM to OpenStack"
     echo -e "     -f <path to SSH public key>: Public SSH key to use to deploy OSM to OpenStack"
     echo -e "     -F <path to cloud-init file>: Cloud-Init userdata file to deploy OSM to OpenStack"
@@ -87,7 +84,6 @@ function usage(){
     echo -e "     [--ha]:                      Installs High Availability bundle. (--charmed option)"
     echo -e "     [--tag]:                     Docker image tag. (--charmed option)"
     echo -e "     [--registry]:                Docker registry with optional credentials as user:pass@hostname:port (--charmed option)"
-
 }
 
 add_repo() {