Add mon-exporter as scrape target in Prometheus

[osm/devops.git] / docker / Prometheus / src / app.py

diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py
index 43cca08..b06f448 100755 (executable)
--- a/docker/Prometheus/src/app.py
+++ b/docker/Prometheus/src/app.py
@@ -37,6 +37,7 @@ from bson import ObjectId
 mongodb_url = os.environ["MONGODB_URL"]
 target_database = os.environ["TARGET_DATABASE"]
 prometheus_config_file = os.environ["PROMETHEUS_CONFIG_FILE"]
+prometheus_base_config_file = os.environ["PROMETHEUS_BASE_CONFIG_FILE"]
 prometheus_url = os.environ["PROMETHEUS_URL"]
 
 
@@ -47,8 +48,7 @@ def get_jobs(client):
 def save_successful_jobs(client, jobs):
     for job in jobs:
         client[target_database].prometheus_jobs.update_one(
-            {"_id": ObjectId(job["_id"]["$oid"])},
-            {"$set": {"is_active": True}}
+            {"_id": ObjectId(job["_id"]["$oid"])}, {"$set": {"is_active": True}}
         )
 
 
@@ -63,23 +63,27 @@ def clean_up_job(prometheus_job):
 
 
 def generate_prometheus_config(prometheus_jobs, config_file_path):
-    config_file = open(config_file_path, encoding="utf-8", mode="r")
-    config_file_contents = config_file.read()
-    config_file.close()
-
-    config_file_yaml = yaml.load(config_file_contents, yaml.FullLoader)
+    with open(config_file_path, encoding="utf-8", mode="r") as config_file:
+        config_file_yaml = yaml.safe_load(config_file)
     if config_file_yaml is None:
         config_file_yaml = {}
-
-    if len(prometheus_jobs) == 0:
+    if "scrape_configs" not in config_file_yaml:
         config_file_yaml["scrape_configs"] = []
-        return config_file_yaml
 
-    config_file_yaml["scrape_configs"] = []
+    prometheus_jobs_to_be_added = []
 
     for prometheus_job in prometheus_jobs:
         cleaned_up_job = clean_up_job(prometheus_job)
-        config_file_yaml["scrape_configs"].append(cleaned_up_job)
+        job_to_be_added = True
+        for sc in config_file_yaml["scrape_configs"]:
+            if sc.get("job_name") == cleaned_up_job.get("job_name"):
+                job_to_be_added = False
+                break
+        if job_to_be_added:
+            prometheus_jobs_to_be_added.append(cleaned_up_job)
+
+    for job in prometheus_jobs_to_be_added:
+        config_file_yaml["scrape_configs"].append(job)
 
     return config_file_yaml
 
@@ -129,18 +133,17 @@ async def main_task(client):
     stored_jobs = get_jobs(client)
     print(f"Jobs detected : {len(stored_jobs):d}")
     generated_prometheus_config = generate_prometheus_config(
-        stored_jobs, prometheus_config_file
+        stored_jobs, prometheus_base_config_file
     )
     print(f"Writing new config file to {prometheus_config_file}")
     config_file = open(prometheus_config_file, "w")
     config_file.truncate(0)
-    config_file.write(yaml.dump(generated_prometheus_config))
+    print(yaml.safe_dump(generated_prometheus_config))
+    config_file.write(yaml.safe_dump(generated_prometheus_config))
     config_file.close()
     print("New config written, updating prometheus")
     update_resp = await reload_prometheus_config(prometheus_url)
-    is_valid = await validate_configuration(
-        prometheus_url, generated_prometheus_config
-    )
+    is_valid = await validate_configuration(prometheus_url, generated_prometheus_config)
     if update_resp and is_valid:
         print("Prometheus config update successful")
         save_successful_jobs(client, stored_jobs)
@@ -153,15 +156,26 @@ async def main_task(client):
 
 async def main():
     client = pymongo.MongoClient(mongodb_url)
-    print("Connected to MongoDB!")
+    print("Created MongoClient to connect to MongoDB!")
 
-    try:
-        print("Refreshing prometheus config file for first time")
-        await main_task(client)
-    except Exception as error:
-        print("Error in first configuration attempt!")
-        print(error)
+    # Initial loop. First refresh of prometheus config file
+    first_refresh_completed = False
+    tries = 1
+    while tries <= 3:
+        try:
+            print("Refreshing prometheus config file for first time")
+            await main_task(client)
+            first_refresh_completed = True
+        except Exception as error:
+            print(f"Error in configuration attempt! Number of tries: {tries}/3")
+            print(error)
+        time.sleep(5)
+        tries += 1
+    if not first_refresh_completed:
+        print("Not possible to refresh prometheus config file for first time")
+        return
 
+    # Main loop
     while True:
         try:
             # Needs mongodb in replica mode as this feature relies in OpLog
@@ -181,6 +195,7 @@ async def main():
             print("Listening to changes in prometheus jobs collection")
             for change in change_stream:
                 print("Change detected, updating prometheus config")
+                print(f"{change}")
                 await main_task(client)
                 print()
         except Exception as error: