X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=blobdiff_plain;f=osm_nbi%2Fsubscriptions.py;h=03bb92b65f4f084b001e18389d65af1b5dc26d4b;hp=8132fe9e7329f96a0fe9738c84f7da2933a6f03c;hb=bee3bad8d15fe0893855d0dff92cef4351629edb;hpb=09e6542ceae8ee08ee7cc372f7344a77bc1f5304 diff --git a/osm_nbi/subscriptions.py b/osm_nbi/subscriptions.py index 8132fe9..03bb92b 100644 --- a/osm_nbi/subscriptions.py +++ b/osm_nbi/subscriptions.py @@ -27,7 +27,7 @@ from http import HTTPStatus from osm_common import dbmongo, dbmemory, msglocal, msgkafka from osm_common.dbbase import DbException from osm_common.msgbase import MsgException -from engine import EngineException +from osm_nbi.engine import EngineException __author__ = "Alfonso Tierno " @@ -48,7 +48,7 @@ class SubscriptionThread(threading.Thread): :param engine: an instance of Engine class, used for deleting instances """ threading.Thread.__init__(self) - + self.to_terminate = False self.config = config self.db = None self.msg = None @@ -57,12 +57,42 @@ class SubscriptionThread(threading.Thread): self.logger = logging.getLogger("nbi.subscriptions") self.aiomain_task = None # asyncio task for receiving kafka bus self.internal_session = { # used for a session to the engine methods - "_id": "subscription", - "id": "subscription", - "project_id": "admin", - "admin": True + "project_id": (), + "set_project": (), + "admin": True, + "force": False, + "public": None, + "method": "delete", } + async def start_kafka(self): + # timeout_wait_for_kafka = 3*60 + kafka_working = True + while not self.to_terminate: + try: + # bug 710 635. The library aiokafka does not recieve anything when the topci at kafka has not been + # created. + # Before subscribe, send dummy messages + await self.msg.aiowrite("admin", "echo", "dummy message", loop=self.loop) + await self.msg.aiowrite("ns", "echo", "dummy message", loop=self.loop) + await self.msg.aiowrite("nsi", "echo", "dummy message", loop=self.loop) + if not kafka_working: + self.logger.critical("kafka is working again") + kafka_working = True + await asyncio.sleep(10, loop=self.loop) + self.aiomain_task = asyncio.ensure_future(self.msg.aioread(("ns", "nsi"), loop=self.loop, + aiocallback=self._msg_callback), + loop=self.loop) + await asyncio.wait_for(self.aiomain_task, timeout=None, loop=self.loop) + except Exception as e: + if self.to_terminate: + return + if kafka_working: + # logging only first time + self.logger.critical("Error accessing kafka '{}'. Retrying ...".format(e)) + kafka_working = False + await asyncio.sleep(10, loop=self.loop) + def run(self): """ Start of the thread @@ -97,22 +127,21 @@ class SubscriptionThread(threading.Thread): raise SubscriptionException(str(e), http_code=e.http_code) self.logger.debug("Starting") - while True: + while not self.to_terminate: try: - self.aiomain_task = asyncio.ensure_future(self.msg.aioread(("ns", "nsi"), loop=self.loop, - callback=self._msg_callback), - loop=self.loop) - self.loop.run_until_complete(self.aiomain_task) - except asyncio.CancelledError: - break # if cancelled it should end, breaking loop + + self.loop.run_until_complete(asyncio.ensure_future(self.start_kafka(), loop=self.loop)) + # except asyncio.CancelledError: + # break # if cancelled it should end, breaking loop except Exception as e: - self.logger.exception("Exception '{}' at messaging read loop".format(e), exc_info=True) + if not self.to_terminate: + self.logger.exception("Exception '{}' at messaging read loop".format(e), exc_info=True) self.logger.debug("Finishing") self._stop() self.loop.close() - def _msg_callback(self, topic, command, params): + async def _msg_callback(self, topic, command, params): """ Callback to process a received message from kafka :param topic: topic received @@ -120,21 +149,27 @@ class SubscriptionThread(threading.Thread): :param params: rest of parameters :return: None """ + msg_to_send = [] try: if topic == "ns": if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"): self.logger.debug("received ns terminated {}".format(params)) if params.get("autoremove"): - self.engine.del_item(self.internal_session, "nsrs", _id=params["nsr_id"]) + self.engine.del_item(self.internal_session, "nsrs", _id=params["nsr_id"], + not_send_msg=msg_to_send) self.logger.debug("ns={} deleted from database".format(params["nsr_id"])) - return - if topic == "nsi": + elif topic == "nsi": if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"): self.logger.debug("received nsi terminated {}".format(params)) if params.get("autoremove"): - self.engine.del_item(self.internal_session, "nsis", _id=params["nsir_id"]) + self.engine.del_item(self.internal_session, "nsis", _id=params["nsir_id"], + not_send_msg=msg_to_send) self.logger.debug("nsis={} deleted from database".format(params["nsir_id"])) - return + + # writing to kafka must be done with our own loop. For this reason it is not allowed Engine to do that, + # but content to be written is stored at msg_to_send + for msg in msg_to_send: + await self.msg.aiowrite(*msg, loop=self.loop) except (EngineException, DbException, MsgException) as e: self.logger.error("Error while processing topic={} command={}: {}".format(topic, command, e)) except Exception as e: @@ -160,4 +195,6 @@ class SubscriptionThread(threading.Thread): but not immediately. :return: None """ - self.loop.call_soon_threadsafe(self.aiomain_task.cancel) + self.to_terminate = True + if self.aiomain_task: + self.loop.call_soon_threadsafe(self.aiomain_task.cancel)