Add init loop in prometheus sidecar container

[osm/devops.git] / docker / Prometheus / src / app.py

diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py
index 157413d..ff29151 100755 (executable)
--- a/docker/Prometheus/src/app.py
+++ b/docker/Prometheus/src/app.py
@@ -48,8 +48,7 @@ def get_jobs(client):
 def save_successful_jobs(client, jobs):
     for job in jobs:
         client[target_database].prometheus_jobs.update_one(
-            {"_id": ObjectId(job["_id"]["$oid"])},
-            {"$set": {"is_active": True}}
+            {"_id": ObjectId(job["_id"]["$oid"])}, {"$set": {"is_active": True}}
         )
 
 
@@ -132,9 +131,7 @@ async def main_task(client):
     config_file.close()
     print("New config written, updating prometheus")
     update_resp = await reload_prometheus_config(prometheus_url)
-    is_valid = await validate_configuration(
-        prometheus_url, generated_prometheus_config
-    )
+    is_valid = await validate_configuration(prometheus_url, generated_prometheus_config)
     if update_resp and is_valid:
         print("Prometheus config update successful")
         save_successful_jobs(client, stored_jobs)
@@ -147,15 +144,26 @@ async def main_task(client):
 
 async def main():
     client = pymongo.MongoClient(mongodb_url)
-    print("Connected to MongoDB!")
+    print("Created MongoClient to connect to MongoDB!")
 
-    try:
-        print("Refreshing prometheus config file for first time")
-        await main_task(client)
-    except Exception as error:
-        print("Error in first configuration attempt!")
-        print(error)
+    # Initial loop. First refresh of prometheus config file
+    first_refresh_completed = False
+    tries = 1
+    while tries <= 3:
+        try:
+            print("Refreshing prometheus config file for first time")
+            await main_task(client)
+            first_refresh_completed = True
+        except Exception as error:
+            print(f"Error in configuration attempt! Number of tries: {tries}/3")
+            print(error)
+        time.sleep(5)
+        tries += 1
+    if not first_refresh_completed:
+        print("Not possible to refresh prometheus config file for first time")
+        return
 
+    # Main loop
     while True:
         try:
             # Needs mongodb in replica mode as this feature relies in OpLog
@@ -175,6 +183,7 @@ async def main():
             print("Listening to changes in prometheus jobs collection")
             for change in change_stream:
                 print("Change detected, updating prometheus config")
+                print(f"{change}")
                 await main_task(client)
                 print()
         except Exception as error: