From 09eaa9263cfd3f0eaf3c10d756a781ee241cb5e6 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Wed, 7 Sep 2022 22:57:53 +0200 Subject: [PATCH] Feature 10961 Addition of external targets to OSM Prometheus This commit allows the automatic regeneration of a Prometheus config file from a sidecar by combining the information in PROMETHEUS_BASE_CONFIG_FILE and the prometheus_jobs in mongo. Change-Id: I0246d60ec32733a088ca8590a6445735549bf4de Signed-off-by: garciadeblas --- docker/Prometheus/Dockerfile | 3 ++- docker/Prometheus/src/app.py | 18 +++++-------- installers/docker/osm_pods/prometheus.yaml | 31 +++++++++++----------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/docker/Prometheus/Dockerfile b/docker/Prometheus/Dockerfile index 7c842077..228f5979 100644 --- a/docker/Prometheus/Dockerfile +++ b/docker/Prometheus/Dockerfile @@ -21,6 +21,7 @@ FROM python:3.8 ENV PROMETHEUS_URL http://prometheus:9090 ENV MONGODB_URL mongodb://mongo:27017 ENV PROMETHEUS_CONFIG_FILE /etc/prometheus/prometheus.yml +ENV PROMETHEUS_BASE_CONFIG_FILE /etc/prometheus_base/prometheus.yml ENV TARGET_DATABASE osm WORKDIR /code @@ -31,4 +32,4 @@ RUN pip install -r requirements.txt COPY src/app.py . -CMD [ "python", "-u", "./app.py" ] \ No newline at end of file +CMD [ "python", "-u", "./app.py" ] diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py index 43cca085..157413dc 100755 --- a/docker/Prometheus/src/app.py +++ b/docker/Prometheus/src/app.py @@ -37,6 +37,7 @@ from bson import ObjectId mongodb_url = os.environ["MONGODB_URL"] target_database = os.environ["TARGET_DATABASE"] prometheus_config_file = os.environ["PROMETHEUS_CONFIG_FILE"] +prometheus_base_config_file = os.environ["PROMETHEUS_BASE_CONFIG_FILE"] prometheus_url = os.environ["PROMETHEUS_URL"] @@ -63,19 +64,12 @@ def clean_up_job(prometheus_job): def generate_prometheus_config(prometheus_jobs, config_file_path): - config_file = open(config_file_path, encoding="utf-8", mode="r") - config_file_contents = config_file.read() - config_file.close() - - config_file_yaml = yaml.load(config_file_contents, yaml.FullLoader) + with open(config_file_path, encoding="utf-8", mode="r") as config_file: + config_file_yaml = yaml.safe_load(config_file) if config_file_yaml is None: config_file_yaml = {} - - if len(prometheus_jobs) == 0: + if "scrape_configs" not in config_file_yaml: config_file_yaml["scrape_configs"] = [] - return config_file_yaml - - config_file_yaml["scrape_configs"] = [] for prometheus_job in prometheus_jobs: cleaned_up_job = clean_up_job(prometheus_job) @@ -129,12 +123,12 @@ async def main_task(client): stored_jobs = get_jobs(client) print(f"Jobs detected : {len(stored_jobs):d}") generated_prometheus_config = generate_prometheus_config( - stored_jobs, prometheus_config_file + stored_jobs, prometheus_base_config_file ) print(f"Writing new config file to {prometheus_config_file}") config_file = open(prometheus_config_file, "w") config_file.truncate(0) - config_file.write(yaml.dump(generated_prometheus_config)) + config_file.write(yaml.safe_dump(generated_prometheus_config)) config_file.close() print("New config written, updating prometheus") update_resp = await reload_prometheus_config(prometheus_url) diff --git a/installers/docker/osm_pods/prometheus.yaml b/installers/docker/osm_pods/prometheus.yaml index ac3a5240..2610710f 100644 --- a/installers/docker/osm_pods/prometheus.yaml +++ b/installers/docker/osm_pods/prometheus.yaml @@ -73,15 +73,12 @@ data: # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. - scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'mon_exporter' - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - - static_configs: - - targets: ['mon:8000'] + scrape_configs: [] + # scrape_configs: + # Add here other external targets, e.g. a pushgateway + # - job_name: 'pushgateway' + # static_configs: + # - targets: ['prometheus-pushgateway:9091'] kind: ConfigMap metadata: name: prom @@ -107,12 +104,12 @@ spec: - name: prometheus-init-config image: busybox command: ["/bin/sh", "-c"] - args: ['if [ ! -f "/etc/prometheus/prometheus.yml" ]; then cp /etc/prometheus_init/prometheus.yml /etc/prometheus; fi'] + args: ['if [ ! -f "/etc/prometheus/prometheus.yml" ]; then cp /config/prometheus.yml /etc/prometheus; fi'] volumeMounts: - name: prom-config mountPath: /etc/prometheus - - name: init-prom-config - mountPath: /etc/prometheus_init + - name: prom-config-base + mountPath: /config containers: - name: prometheus image: prom/prometheus:v2.28.1 @@ -134,6 +131,8 @@ spec: value: mongodb://mongodb-k8s:27017/?replicaSet=rs0 - name: PROMETHEUS_CONFIG_FILE value: /etc/prometheus/prometheus.yml + - name: PROMETHEUS_BASE_CONFIG_FILE + value: /etc/prometheus_base/prometheus.yml - name: TARGET_DATABASE value: osm - name: PROMETHEUS_URL @@ -141,11 +140,13 @@ spec: volumeMounts: - name: prom-config mountPath: /etc/prometheus + - name: prom-config-base + mountPath: /etc/prometheus_base volumes: - - emptyDir: {} - name: prom-db + - name: prom-db + emptyDir: {} - name: prom-config emptyDir: {} - - name: init-prom-config + - name: prom-config-base configMap: name: prom -- 2.17.1