From 946821f0dcc61f0bf0fe6d78bc2dc7db2636296c Mon Sep 17 00:00:00 2001 From: bravof Date: Thu, 11 Jun 2020 18:50:32 -0400 Subject: [PATCH] fix(healthcheck): new flag implemented as a file for health checking in POL (it only appears after all migrations and connections were made) Change-Id: I7fac1e4f219526b9ac7a4e28f1be1b456d365b5d Signed-off-by: bravof --- docker/Dockerfile | 6 +++--- osm_policy_module/cmd/policy_module_agent.py | 5 +++++ osm_policy_module/cmd/policy_module_healthcheck.py | 8 +++++++- osm_policy_module/core/agent.py | 6 ++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a3cb316..7045b0f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,9 +20,9 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -FROM ubuntu:16.04 +FROM ubuntu:20.04 -LABEL authors="Benjamín Díaz" +LABEL authors="Benjamín Díaz, Fabián Bravo" RUN apt-get --yes update \ && apt-get --yes install python3 python3-pip libmysqlclient-dev git mysql-client \ @@ -47,7 +47,7 @@ ENV OSMPOL_SQL_DATABASE_URI sqlite:///policy_module.db ENV OSMPOL_GLOBAL_LOGLEVEL INFO -HEALTHCHECK --interval=5s --timeout=2s --retries=12 \ +HEALTHCHECK --interval=10s --timeout=5s --retries=10 --start-period=30s \ CMD osm-pol-healthcheck || exit 1 CMD /bin/bash /policy_module/docker/scripts/start.sh \ No newline at end of file diff --git a/osm_policy_module/cmd/policy_module_agent.py b/osm_policy_module/cmd/policy_module_agent.py index af2f602..e6c0681 100644 --- a/osm_policy_module/cmd/policy_module_agent.py +++ b/osm_policy_module/cmd/policy_module_agent.py @@ -25,6 +25,7 @@ import argparse import asyncio import logging import sys +import os from osm_policy_module.core.agent import PolicyModuleAgent from osm_policy_module.core.config import Config @@ -32,6 +33,10 @@ from osm_policy_module.core.database import DatabaseManager def main(): + # Cleanup old temp health file + if os.path.exists('/tmp/osm_pol_agent_health_flag'): + os.remove('/tmp/osm_pol_agent_health_flag') + parser = argparse.ArgumentParser(prog='osm-policy-agent') parser.add_argument('--config-file', nargs='?', help='POL configuration file') args = parser.parse_args() diff --git a/osm_policy_module/cmd/policy_module_healthcheck.py b/osm_policy_module/cmd/policy_module_healthcheck.py index f184afd..de85ad0 100644 --- a/osm_policy_module/cmd/policy_module_healthcheck.py +++ b/osm_policy_module/cmd/policy_module_healthcheck.py @@ -23,6 +23,7 @@ import argparse import logging import subprocess import sys +import os log = logging.getLogger(__name__) @@ -52,7 +53,12 @@ def _processes_running(): if not _contains_process(processes_running, p): log.error("Process %s not running!" % p) return False - return True + + # Check if process is running properly (listening to kafka bus) + if os.path.exists('/tmp/osm_pol_agent_health_flag'): + return True + else: + return False if __name__ == '__main__': diff --git a/osm_policy_module/core/agent.py b/osm_policy_module/core/agent.py index 34b852a..3f87d16 100644 --- a/osm_policy_module/core/agent.py +++ b/osm_policy_module/core/agent.py @@ -23,6 +23,8 @@ ## import asyncio import logging +from pathlib import Path +import os import peewee @@ -52,14 +54,18 @@ class PolicyModuleAgent: self.loop.run_until_complete(self.start()) async def start(self): + Path('/tmp/osm_pol_agent_health_flag').touch() topics = [ "ns", "alarm_response" ] await self.msg_bus.aioread(topics, self._process_msg) log.critical("Exiting...") + if os.path.exists('/tmp/osm_pol_agent_health_flag'): + os.remove('/tmp/osm_pol_agent_health_flag') async def _process_msg(self, topic, key, msg): + Path('/tmp/osm_pol_agent_health_flag').touch() log.debug("_process_msg topic=%s key=%s msg=%s", topic, key, msg) try: if key in ALLOWED_KAFKA_KEYS: -- 2.17.1