X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=blobdiff_plain;f=osm_nbi%2Fsubscriptions.py;h=03bb92b65f4f084b001e18389d65af1b5dc26d4b;hp=557bf800f0915d4fa512c39e92396750d7bf7ae9;hb=bee3bad8d15fe0893855d0dff92cef4351629edb;hpb=86e916adddf8831299ca21a0b420dddaa632ddb6;ds=sidebyside

diff --git a/osm_nbi/subscriptions.py b/osm_nbi/subscriptions.py
index 557bf80..03bb92b 100644
--- a/osm_nbi/subscriptions.py
+++ b/osm_nbi/subscriptions.py
@@ -27,7 +27,7 @@ from http import HTTPStatus
 from osm_common import dbmongo, dbmemory, msglocal, msgkafka
 from osm_common.dbbase import DbException
 from osm_common.msgbase import MsgException
-from engine import EngineException
+from osm_nbi.engine import EngineException
 
 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 
@@ -65,6 +65,34 @@ class SubscriptionThread(threading.Thread):
             "method": "delete",
         }
 
+    async def start_kafka(self):
+        # timeout_wait_for_kafka = 3*60
+        kafka_working = True
+        while not self.to_terminate:
+            try:
+                # bug 710 635. The library aiokafka does not recieve anything when the topci at kafka has not been
+                # created.
+                # Before subscribe, send dummy messages
+                await self.msg.aiowrite("admin", "echo", "dummy message", loop=self.loop)
+                await self.msg.aiowrite("ns", "echo", "dummy message", loop=self.loop)
+                await self.msg.aiowrite("nsi", "echo", "dummy message", loop=self.loop)
+                if not kafka_working:
+                    self.logger.critical("kafka is working again")
+                    kafka_working = True
+                await asyncio.sleep(10, loop=self.loop)
+                self.aiomain_task = asyncio.ensure_future(self.msg.aioread(("ns", "nsi"), loop=self.loop,
+                                                                           aiocallback=self._msg_callback),
+                                                          loop=self.loop)
+                await asyncio.wait_for(self.aiomain_task, timeout=None, loop=self.loop)
+            except Exception as e:
+                if self.to_terminate:
+                    return
+                if kafka_working:
+                    # logging only first time
+                    self.logger.critical("Error accessing kafka '{}'. Retrying ...".format(e))
+                    kafka_working = False
+            await asyncio.sleep(10, loop=self.loop)
+
     def run(self):
         """
         Start of the thread
@@ -101,10 +129,8 @@ class SubscriptionThread(threading.Thread):
         self.logger.debug("Starting")
         while not self.to_terminate:
             try:
-                self.aiomain_task = asyncio.ensure_future(self.msg.aioread(("ns", "nsi"), loop=self.loop,
-                                                                           callback=self._msg_callback),
-                                                          loop=self.loop)
-                self.loop.run_until_complete(self.aiomain_task)
+
+                self.loop.run_until_complete(asyncio.ensure_future(self.start_kafka(), loop=self.loop))
             # except asyncio.CancelledError:
             #     break  # if cancelled it should end, breaking loop
             except Exception as e:
@@ -115,7 +141,7 @@ class SubscriptionThread(threading.Thread):
         self._stop()
         self.loop.close()
 
-    def _msg_callback(self, topic, command, params):
+    async def _msg_callback(self, topic, command, params):
         """
         Callback to process a received message from kafka
         :param topic:  topic received
@@ -123,21 +149,27 @@ class SubscriptionThread(threading.Thread):
         :param params: rest of parameters
         :return: None
         """
+        msg_to_send = []
         try:
             if topic == "ns":
                 if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"):
                     self.logger.debug("received ns terminated {}".format(params))
                     if params.get("autoremove"):
-                        self.engine.del_item(self.internal_session, "nsrs", _id=params["nsr_id"])
+                        self.engine.del_item(self.internal_session, "nsrs", _id=params["nsr_id"],
+                                             not_send_msg=msg_to_send)
                         self.logger.debug("ns={} deleted from database".format(params["nsr_id"]))
-                    return
-            if topic == "nsi":
+            elif topic == "nsi":
                 if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"):
                     self.logger.debug("received nsi terminated {}".format(params))
                     if params.get("autoremove"):
-                        self.engine.del_item(self.internal_session, "nsis", _id=params["nsir_id"])
+                        self.engine.del_item(self.internal_session, "nsis", _id=params["nsir_id"],
+                                             not_send_msg=msg_to_send)
                         self.logger.debug("nsis={} deleted from database".format(params["nsir_id"]))
-                    return
+
+            # writing to kafka must be done with our own loop. For this reason it is not allowed Engine to do that, 
+            # but content to be written is stored at msg_to_send
+            for msg in msg_to_send:
+                await self.msg.aiowrite(*msg, loop=self.loop)
         except (EngineException, DbException, MsgException) as e:
             self.logger.error("Error while processing topic={} command={}: {}".format(topic, command, e))
         except Exception as e:
@@ -164,4 +196,5 @@ class SubscriptionThread(threading.Thread):
         :return: None
         """
         self.to_terminate = True
-        self.loop.call_soon_threadsafe(self.aiomain_task.cancel)
+        if self.aiomain_task:
+            self.loop.call_soon_threadsafe(self.aiomain_task.cancel)