X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=docker%2FPrometheus%2Fsrc%2Fapp.py;h=fc1e2bb10cb806b9e19a3fdc2d44e547ff6078d0;hb=feb9212ef9c27a5d1a972cfbb7b5a829220c0699;hp=78e70152e045fc13638f3131426a23856edb714a;hpb=c973b57b1f34ecb64f248f3098ac7b79c1b8a0ae;p=osm%2Fdevops.git

diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py
index 78e70152..fc1e2bb1 100755
--- a/docker/Prometheus/src/app.py
+++ b/docker/Prometheus/src/app.py
@@ -29,6 +29,7 @@ import aiohttp
 import asyncio
 import copy
 import json
+import time
 from bson.json_util import dumps
 from bson import ObjectId
 
@@ -138,21 +139,32 @@ async def main():
   client = pymongo.MongoClient(mongodb_url)
   print('Connected to MongoDB!')
 
-  print('Refreshing prometheus config file for first time')
-  await main_task(client)
-
-  #Needs mongodb in replica mode as this feature relies in OpLog
-  change_stream = client[target_database].prometheus_jobs.watch([{
-    '$match': {
-      #If you want to modify a particular job, delete and insert it again
-      'operationType': { '$in': ['insert', 'delete'] }
-    }
-  }])
-
-  #Single thread, no race conditions and ops are queued up in order
-  for change in change_stream:
-    print("Change detected, updating prometheus config")
+  try:
+    print('Refreshing prometheus config file for first time')
     await main_task(client)
-    print()
+  except Exception as error:
+    print("Error in first configuration attempt!")
+    print(error)
+
+  while(True):
+    try:
+      #Needs mongodb in replica mode as this feature relies in OpLog
+      change_stream = client[target_database].prometheus_jobs.watch([{
+        '$match': {
+          #If you want to modify a particular job, delete and insert it again
+          'operationType': { '$in': ['insert', 'delete'] }
+        }
+      }])
+
+      #Single thread, no race conditions and ops are queued up in order
+      print("Listening to changes in prometheus jobs collection")
+      for change in change_stream:
+        print("Change detected, updating prometheus config")
+        await main_task(client)
+        print()
+    except Exception as error:
+      print(error)
+    print("Detected failure while listening to prometheus jobs collection, retrying...")
+    time.sleep(5) 
 
 asyncio.run(main())