From: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Wed, 7 Sep 2022 20:57:53 +0000 (+0200)
Subject: Feature 10961 Addition of external targets to OSM Prometheus
X-Git-Tag: release-v13.0-start~18
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=09eaa9263cfd3f0eaf3c10d756a781ee241cb5e6;p=osm%2Fdevops.git

Feature 10961 Addition of external targets to OSM Prometheus

This commit allows the automatic regeneration of a Prometheus config
file from a sidecar by combining the information in PROMETHEUS_BASE_CONFIG_FILE
and the prometheus_jobs in mongo.

Change-Id: I0246d60ec32733a088ca8590a6445735549bf4de
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
---

diff --git a/docker/Prometheus/Dockerfile b/docker/Prometheus/Dockerfile
index 7c842077..228f5979 100644
--- a/docker/Prometheus/Dockerfile
+++ b/docker/Prometheus/Dockerfile
@@ -21,6 +21,7 @@ FROM python:3.8
 ENV PROMETHEUS_URL http://prometheus:9090
 ENV MONGODB_URL mongodb://mongo:27017
 ENV PROMETHEUS_CONFIG_FILE /etc/prometheus/prometheus.yml
+ENV PROMETHEUS_BASE_CONFIG_FILE /etc/prometheus_base/prometheus.yml
 ENV TARGET_DATABASE osm
 
 WORKDIR /code
@@ -31,4 +32,4 @@ RUN pip install -r requirements.txt
 
 COPY src/app.py .
 
-CMD [ "python", "-u", "./app.py" ]
\ No newline at end of file
+CMD [ "python", "-u", "./app.py" ]
diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py
index 43cca085..157413dc 100755
--- a/docker/Prometheus/src/app.py
+++ b/docker/Prometheus/src/app.py
@@ -37,6 +37,7 @@ from bson import ObjectId
 mongodb_url = os.environ["MONGODB_URL"]
 target_database = os.environ["TARGET_DATABASE"]
 prometheus_config_file = os.environ["PROMETHEUS_CONFIG_FILE"]
+prometheus_base_config_file = os.environ["PROMETHEUS_BASE_CONFIG_FILE"]
 prometheus_url = os.environ["PROMETHEUS_URL"]
 
 
@@ -63,19 +64,12 @@ def clean_up_job(prometheus_job):
 
 
 def generate_prometheus_config(prometheus_jobs, config_file_path):
-    config_file = open(config_file_path, encoding="utf-8", mode="r")
-    config_file_contents = config_file.read()
-    config_file.close()
-
-    config_file_yaml = yaml.load(config_file_contents, yaml.FullLoader)
+    with open(config_file_path, encoding="utf-8", mode="r") as config_file:
+        config_file_yaml = yaml.safe_load(config_file)
     if config_file_yaml is None:
         config_file_yaml = {}
-
-    if len(prometheus_jobs) == 0:
+    if "scrape_configs" not in config_file_yaml:
         config_file_yaml["scrape_configs"] = []
-        return config_file_yaml
-
-    config_file_yaml["scrape_configs"] = []
 
     for prometheus_job in prometheus_jobs:
         cleaned_up_job = clean_up_job(prometheus_job)
@@ -129,12 +123,12 @@ async def main_task(client):
     stored_jobs = get_jobs(client)
     print(f"Jobs detected : {len(stored_jobs):d}")
     generated_prometheus_config = generate_prometheus_config(
-        stored_jobs, prometheus_config_file
+        stored_jobs, prometheus_base_config_file
     )
     print(f"Writing new config file to {prometheus_config_file}")
     config_file = open(prometheus_config_file, "w")
     config_file.truncate(0)
-    config_file.write(yaml.dump(generated_prometheus_config))
+    config_file.write(yaml.safe_dump(generated_prometheus_config))
     config_file.close()
     print("New config written, updating prometheus")
     update_resp = await reload_prometheus_config(prometheus_url)
diff --git a/installers/docker/osm_pods/prometheus.yaml b/installers/docker/osm_pods/prometheus.yaml
index ac3a5240..2610710f 100644
--- a/installers/docker/osm_pods/prometheus.yaml
+++ b/installers/docker/osm_pods/prometheus.yaml
@@ -73,15 +73,12 @@ data:
 
     # A scrape configuration containing exactly one endpoint to scrape:
     # Here it's Prometheus itself.
-    scrape_configs:
-      # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-      - job_name: 'mon_exporter'
-
-        # metrics_path defaults to '/metrics'
-        # scheme defaults to 'http'.
-
-        static_configs:
-        - targets: ['mon:8000']
+    scrape_configs: []
+    # scrape_configs:
+      # Add here other external targets, e.g. a pushgateway
+      # - job_name: 'pushgateway'
+      #   static_configs:
+      #   - targets: ['prometheus-pushgateway:9091']
 kind: ConfigMap
 metadata:
   name: prom
@@ -107,12 +104,12 @@ spec:
       - name: prometheus-init-config
         image: busybox
         command: ["/bin/sh", "-c"]
-        args: ['if [ ! -f "/etc/prometheus/prometheus.yml" ]; then cp /etc/prometheus_init/prometheus.yml /etc/prometheus; fi']
+        args: ['if [ ! -f "/etc/prometheus/prometheus.yml" ]; then cp /config/prometheus.yml /etc/prometheus; fi']
         volumeMounts:
           - name: prom-config
             mountPath: /etc/prometheus
-          - name: init-prom-config
-            mountPath: /etc/prometheus_init
+          - name: prom-config-base
+            mountPath: /config
       containers:
       - name: prometheus
         image: prom/prometheus:v2.28.1
@@ -134,6 +131,8 @@ spec:
           value: mongodb://mongodb-k8s:27017/?replicaSet=rs0
         - name: PROMETHEUS_CONFIG_FILE
           value: /etc/prometheus/prometheus.yml
+        - name: PROMETHEUS_BASE_CONFIG_FILE
+          value: /etc/prometheus_base/prometheus.yml
         - name: TARGET_DATABASE
           value: osm
         - name: PROMETHEUS_URL
@@ -141,11 +140,13 @@ spec:
         volumeMounts:
         - name: prom-config
           mountPath: /etc/prometheus
+        - name: prom-config-base
+          mountPath: /etc/prometheus_base
       volumes:
-      - emptyDir: {}
-        name: prom-db
+      - name: prom-db
+        emptyDir: {}
       - name: prom-config
         emptyDir: {}
-      - name: init-prom-config
+      - name: prom-config-base
         configMap:
           name: prom