- client = pymongo.MongoClient(mongodb_url)
- print('Connected to MongoDB!')
-
- try:
- print('Refreshing prometheus config file for first time')
- await main_task(client)
- except Exception as error:
- print("Error in first configuration attempt!")
- print(error)
-
- while(True):
- try:
- #Needs mongodb in replica mode as this feature relies in OpLog
- change_stream = client[target_database].prometheus_jobs.watch([{
- '$match': {
- #If you want to modify a particular job, delete and insert it again
- 'operationType': { '$in': ['insert', 'delete'] }
- }
- }])
-
- #Single thread, no race conditions and ops are queued up in order
- print("Listening to changes in prometheus jobs collection")
- for change in change_stream:
- print("Change detected, updating prometheus config")
- await main_task(client)
- print()
- except Exception as error:
- print(error)
- print("Detected failure while listening to prometheus jobs collection, retrying...")
- time.sleep(5)
+ client = pymongo.MongoClient(mongodb_url)
+ print("Created MongoClient to connect to MongoDB!")
+
+ # Initial loop. First refresh of prometheus config file
+ first_refresh_completed = False
+ tries = 1
+ while tries <= 3:
+ try:
+ print("Refreshing prometheus config file for first time")
+ await main_task(client)
+ first_refresh_completed = True
+ except Exception as error:
+ print(f"Error in configuration attempt! Number of tries: {tries}/3")
+ print(error)
+ time.sleep(5)
+ tries += 1
+ if not first_refresh_completed:
+ print("Not possible to refresh prometheus config file for first time")
+ return
+
+ # Main loop
+ while True:
+ try:
+ # Needs mongodb in replica mode as this feature relies in OpLog
+ change_stream = client[target_database].prometheus_jobs.watch(
+ [
+ {
+ "$match": {
+ # If you want to modify a particular job,
+ # delete and insert it again
+ "operationType": {"$in": ["insert", "delete"]}
+ }
+ }
+ ]
+ )
+
+ # Single thread, no race conditions and ops are queued up in order
+ print("Listening to changes in prometheus jobs collection")
+ for change in change_stream:
+ print("Change detected, updating prometheus config")
+ print(f"{change}")
+ await main_task(client)
+ print()
+ except Exception as error:
+ print(error)
+ print(
+ "Detected failure while listening to prometheus jobs collection, "
+ "retrying..."
+ )
+ time.sleep(5)
+