X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=blobdiff_plain;f=osm_nbi%2Fsubscriptions.py;h=6810ccd6a0a37a159e37cbda9d5618f0c8480643;hp=ec70d0c2755cb0405facbd8df0b0a19b5c4e03a0;hb=4568a372eb5a204e04d917213de03ec51f9110c1;hpb=786aeddab7b7491a139c958ecec6439bee5da83a diff --git a/osm_nbi/subscriptions.py b/osm_nbi/subscriptions.py index ec70d0c..6810ccd 100644 --- a/osm_nbi/subscriptions.py +++ b/osm_nbi/subscriptions.py @@ -35,14 +35,12 @@ __author__ = "Alfonso Tierno " class SubscriptionException(Exception): - def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST): self.http_code = http_code Exception.__init__(self, message) class SubscriptionThread(threading.Thread): - def __init__(self, config, engine): """ Constructor of class @@ -57,8 +55,12 @@ class SubscriptionThread(threading.Thread): self.engine = engine self.loop = None self.logger = logging.getLogger("nbi.subscriptions") - self.aiomain_task_admin = None # asyncio task for receiving admin actions from kafka bus - self.aiomain_task = None # asyncio task for receiving normal actions from kafka bus + self.aiomain_task_admin = ( + None # asyncio task for receiving admin actions from kafka bus + ) + self.aiomain_task = ( + None # asyncio task for receiving normal actions from kafka bus + ) self.internal_session = { # used for a session to the engine methods "project_id": (), "set_project": (), @@ -77,7 +79,9 @@ class SubscriptionThread(threading.Thread): # bug 710 635. The library aiokafka does not recieve anything when the topci at kafka has not been # created. # Before subscribe, send dummy messages - await self.msg.aiowrite("admin", "echo", "dummy message", loop=self.loop) + await self.msg.aiowrite( + "admin", "echo", "dummy message", loop=self.loop + ) await self.msg.aiowrite("ns", "echo", "dummy message", loop=self.loop) await self.msg.aiowrite("nsi", "echo", "dummy message", loop=self.loop) if not kafka_working: @@ -86,26 +90,44 @@ class SubscriptionThread(threading.Thread): if not self.aiomain_task_admin: await asyncio.sleep(10, loop=self.loop) self.logger.debug("Starting admin subscription task") - self.aiomain_task_admin = asyncio.ensure_future(self.msg.aioread(("admin",), loop=self.loop, - group_id=False, - aiocallback=self._msg_callback), - loop=self.loop) + self.aiomain_task_admin = asyncio.ensure_future( + self.msg.aioread( + ("admin",), + loop=self.loop, + group_id=False, + aiocallback=self._msg_callback, + ), + loop=self.loop, + ) if not self.aiomain_task: await asyncio.sleep(10, loop=self.loop) self.logger.debug("Starting non-admin subscription task") - self.aiomain_task = asyncio.ensure_future(self.msg.aioread(("ns", "nsi"), loop=self.loop, - aiocallback=self._msg_callback), - loop=self.loop) - done, _ = await asyncio.wait([self.aiomain_task, self.aiomain_task_admin], - timeout=None, loop=self.loop, return_when=asyncio.FIRST_COMPLETED) + self.aiomain_task = asyncio.ensure_future( + self.msg.aioread( + ("ns", "nsi"), + loop=self.loop, + aiocallback=self._msg_callback, + ), + loop=self.loop, + ) + done, _ = await asyncio.wait( + [self.aiomain_task, self.aiomain_task_admin], + timeout=None, + loop=self.loop, + return_when=asyncio.FIRST_COMPLETED, + ) try: if self.aiomain_task_admin in done: exc = self.aiomain_task_admin.exception() - self.logger.error("admin subscription task exception: {}".format(exc)) + self.logger.error( + "admin subscription task exception: {}".format(exc) + ) self.aiomain_task_admin = None if self.aiomain_task in done: exc = self.aiomain_task.exception() - self.logger.error("non-admin subscription task exception: {}".format(exc)) + self.logger.error( + "non-admin subscription task exception: {}".format(exc) + ) self.aiomain_task = None except asyncio.CancelledError: pass @@ -114,7 +136,9 @@ class SubscriptionThread(threading.Thread): return if kafka_working: # logging only first time - self.logger.critical("Error accessing kafka '{}'. Retrying ...".format(e)) + self.logger.critical( + "Error accessing kafka '{}'. Retrying ...".format(e) + ) kafka_working = False await asyncio.sleep(10, loop=self.loop) @@ -133,8 +157,11 @@ class SubscriptionThread(threading.Thread): self.db = dbmemory.DbMemory() self.db.db_connect(self.config["database"]) else: - raise SubscriptionException("Invalid configuration param '{}' at '[database]':'driver'".format( - self.config["database"]["driver"])) + raise SubscriptionException( + "Invalid configuration param '{}' at '[database]':'driver'".format( + self.config["database"]["driver"] + ) + ) if not self.msg: config_msg = self.config["message"].copy() config_msg["loop"] = self.loop @@ -145,8 +172,11 @@ class SubscriptionThread(threading.Thread): self.msg = msgkafka.MsgKafka() self.msg.connect(config_msg) else: - raise SubscriptionException("Invalid configuration param '{}' at '[message]':'driver'".format( - config_msg["driver"])) + raise SubscriptionException( + "Invalid configuration param '{}' at '[message]':'driver'".format( + config_msg["driver"] + ) + ) self.nslcm = NsLcmNotification(self.db) except (DbException, MsgException) as e: raise SubscriptionException(str(e), http_code=e.http_code) @@ -155,12 +185,16 @@ class SubscriptionThread(threading.Thread): while not self.to_terminate: try: - self.loop.run_until_complete(asyncio.ensure_future(self.start_kafka(), loop=self.loop)) + self.loop.run_until_complete( + asyncio.ensure_future(self.start_kafka(), loop=self.loop) + ) # except asyncio.CancelledError: # break # if cancelled it should end, breaking loop except Exception as e: if not self.to_terminate: - self.logger.exception("Exception '{}' at messaging read loop".format(e), exc_info=True) + self.logger.exception( + "Exception '{}' at messaging read loop".format(e), exc_info=True + ) self.logger.debug("Finishing") self._stop() @@ -177,69 +211,125 @@ class SubscriptionThread(threading.Thread): msg_to_send = [] try: if topic == "ns": - if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"): + if command == "terminated" and params["operationState"] in ( + "COMPLETED", + "PARTIALLY_COMPLETED", + ): self.logger.debug("received ns terminated {}".format(params)) if params.get("autoremove"): - self.engine.del_item(self.internal_session, "nsrs", _id=params["nsr_id"], - not_send_msg=msg_to_send) - self.logger.debug("ns={} deleted from database".format(params["nsr_id"])) + self.engine.del_item( + self.internal_session, + "nsrs", + _id=params["nsr_id"], + not_send_msg=msg_to_send, + ) + self.logger.debug( + "ns={} deleted from database".format(params["nsr_id"]) + ) # Check for nslcm notification if isinstance(params, dict): # Check availability of operationState and command - if (not params.get("operationState")) or (not command) or (not params.get("operationParams")): - self.logger.debug("Message can not be used for notification of nslcm") + if ( + (not params.get("operationState")) + or (not command) + or (not params.get("operationParams")) + ): + self.logger.debug( + "Message can not be used for notification of nslcm" + ) else: nsd_id = params["operationParams"].get("nsdId") ns_instance_id = params["operationParams"].get("nsInstanceId") # Any one among nsd_id, ns_instance_id should be present. if not (nsd_id or ns_instance_id): - self.logger.debug("Message can not be used for notification of nslcm") + self.logger.debug( + "Message can not be used for notification of nslcm" + ) else: op_state = params["operationState"] - event_details = {"topic": topic, "command": command.upper(), "params": params} - subscribers = self.nslcm.get_subscribers(nsd_id, ns_instance_id, command.upper(), op_state, - event_details) + event_details = { + "topic": topic, + "command": command.upper(), + "params": params, + } + subscribers = self.nslcm.get_subscribers( + nsd_id, + ns_instance_id, + command.upper(), + op_state, + event_details, + ) # self.logger.debug("subscribers list: ") # self.logger.debug(subscribers) if subscribers: - asyncio.ensure_future(self.nslcm.send_notifications(subscribers, loop=self.loop), - loop=self.loop) + asyncio.ensure_future( + self.nslcm.send_notifications( + subscribers, loop=self.loop + ), + loop=self.loop, + ) else: - self.logger.debug("Message can not be used for notification of nslcm") + self.logger.debug( + "Message can not be used for notification of nslcm" + ) elif topic == "nsi": - if command == "terminated" and params["operationState"] in ("COMPLETED", "PARTIALLY_COMPLETED"): + if command == "terminated" and params["operationState"] in ( + "COMPLETED", + "PARTIALLY_COMPLETED", + ): self.logger.debug("received nsi terminated {}".format(params)) if params.get("autoremove"): - self.engine.del_item(self.internal_session, "nsis", _id=params["nsir_id"], - not_send_msg=msg_to_send) - self.logger.debug("nsis={} deleted from database".format(params["nsir_id"])) + self.engine.del_item( + self.internal_session, + "nsis", + _id=params["nsir_id"], + not_send_msg=msg_to_send, + ) + self.logger.debug( + "nsis={} deleted from database".format(params["nsir_id"]) + ) elif topic == "admin": self.logger.debug("received {} {} {}".format(topic, command, params)) - if command in ["echo", "ping"]: # ignored commands + if command in ["echo", "ping"]: # ignored commands pass elif command == "revoke_token": if params: if isinstance(params, dict) and "_id" in params: tid = params.get("_id") self.engine.authenticator.tokens_cache.pop(tid, None) - self.logger.debug("token '{}' removed from token_cache".format(tid)) + self.logger.debug( + "token '{}' removed from token_cache".format(tid) + ) else: - self.logger.debug("unrecognized params in command '{} {}': {}" - .format(topic, command, params)) + self.logger.debug( + "unrecognized params in command '{} {}': {}".format( + topic, command, params + ) + ) else: self.engine.authenticator.tokens_cache.clear() self.logger.debug("token_cache cleared") else: - self.logger.debug("unrecognized command '{} {}'".format(topic, command)) + self.logger.debug( + "unrecognized command '{} {}'".format(topic, command) + ) # writing to kafka must be done with our own loop. For this reason it is not allowed Engine to do that, # but content to be written is stored at msg_to_send for msg in msg_to_send: await self.msg.aiowrite(*msg, loop=self.loop) except (EngineException, DbException, MsgException) as e: - self.logger.error("Error while processing topic={} command={}: {}".format(topic, command, e)) + self.logger.error( + "Error while processing topic={} command={}: {}".format( + topic, command, e + ) + ) except Exception as e: - self.logger.exception("Exception while processing topic={} command={}: {}".format(topic, command, e), - exc_info=True) + self.logger.exception( + "Exception while processing topic={} command={}: {}".format( + topic, command, e + ), + exc_info=True, + ) def _stop(self): """