Fix 2188: Added config parameter to disable vms status check

[osm/MON.git] / osm_mon / dashboarder / dashboarder.py

diff --git a/osm_mon/dashboarder/dashboarder.py b/osm_mon/dashboarder/dashboarder.py
index bed157a..f3bd97e 100644 (file)
--- a/osm_mon/dashboarder/dashboarder.py
+++ b/osm_mon/dashboarder/dashboarder.py
@@ -47,14 +47,19 @@ class Dashboarder:
     def run(self):
         self.loop.run_until_complete(self.start())
 
-    async def start(self):
+    async def start(self, wait_time=5):
         topics = ["users", "project"]
-        try:
-            await self.msg_bus.aioread(topics, self._user_msg)
-        except Exception as e:
-            # Failed to subscribe to kafka topics
-            log.error("Error when subscribing to topics %s", str(topics))
-            log.exception("Exception %s", str(e))
+        while True:
+            try:
+                await self.msg_bus.aioread(topics, self._user_msg)
+                break
+            except Exception as e:
+                # Failed to subscribe to kafka topics
+                log.error("Error when subscribing to topic(s) %s", str(topics))
+                log.exception("Exception %s", str(e))
+                # Wait for some time for kaka to stabilize and then reattempt to subscribe again
+                time.sleep(wait_time)
+                log.info("Retrying to subscribe the kafka topic(s) %s", str(topics))
 
     async def _user_msg(self, topic, key, values):
         log.debug(
@@ -141,6 +146,7 @@ class Dashboarder:
                 time.sleep(int(self.conf.get("dashboarder", "interval")))
                 continue
             try:
+                self.grafana_cleanup()
                 self.create_dashboards()
                 time.sleep(int(self.conf.get("dashboarder", "interval")))
             except Exception:
@@ -149,3 +155,10 @@ class Dashboarder:
     def create_dashboards(self):
         self.service.create_dashboards()
         log.debug("Dashboarder Service > create_dashboards called!")
+
+    def grafana_cleanup(self):
+        # Cleaning up non existing users from grafana
+        self.service.delete_non_existing_users()
+        # TODO
+        # Cleanup of teams from grafana
+        log.debug("Deleted non existing users from dashbaorder service")