X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_lcm%2Fprometheus.py;h=2be94afc99d8ea2a37ba1bd2d46879948ff4790d;hb=aae391fc;hp=c1f49b08a9fb746b3c04c292f70f0bbbe8b717ee;hpb=b996d94c0f74b1922d7f93a51ec328c6f370ff86;p=osm%2FLCM.git diff --git a/osm_lcm/prometheus.py b/osm_lcm/prometheus.py index c1f49b0..2be94af 100644 --- a/osm_lcm/prometheus.py +++ b/osm_lcm/prometheus.py @@ -101,10 +101,10 @@ class Prometheus: :param remove_jobs: list with jobs to remove [job_id_1, job_id_2] :return: result. If false prometheus denies this configuration. Exception on error """ - for retry in range(4): + for retry in range(20): result = True if retry: # first time do not wait - await asyncio.sleep(self.PROMETHEUS_LOCKED_TIME / 2, loop=self.loop) + await asyncio.sleep(4 + retry, loop=self.loop) # lock database now = time() @@ -132,9 +132,10 @@ class Prometheus: for job_id in remove_jobs: prometheus_data["scrape_configs"].pop(job_id, None) pull_dict = {"scrape_configs." + job_id: None for job_id in remove_jobs} - self.logger.debug(". ".join(log_text_list)) + self.logger.debug("Updating. " + ". ".join(log_text_list)) if not await self.send_data(prometheus_data): + self.logger.error("Cannot update add_jobs: {}. remove_jobs: {}".format(add_jobs, remove_jobs)) push_dict = pull_dict = None result = False @@ -155,7 +156,7 @@ class Prometheus: del new_config["_id"] del new_config["_admin"] new_scrape_configs = [] - + # generate a list with the values of scrape_configs for scrape_config in new_config["scrape_configs"].values(): scrape_config = scrape_config.copy() @@ -190,7 +191,7 @@ class Prometheus: restore_backup = False return True except Exception as e: - self.logger.error("Error updating prometheus configuration url={}: {}".format(self.server, e)) + self.logger.error("Error updating configuration url={}: {}".format(self.server, e)) return False finally: if restore_backup: @@ -206,9 +207,14 @@ class Prometheus: current_config_yaml = yaml.safe_load(current_config['data']['yaml']) current_jobs = [j["job_name"] for j in current_config_yaml["scrape_configs"]] expected_jobs = [j["job_name"] for j in expected_config["scrape_configs"]] - return current_jobs == expected_jobs + if current_jobs == expected_jobs: + return True + else: + self.logger.error("Not all jobs have been loaded. Target jobs: {} Loaded jobs: {}".format( + expected_jobs, current_jobs)) + return False except Exception as e: - self.logger.error("Invalid obtained prometheus status. Error: '{}'. Obtained data: '{}'".format( + self.logger.error("Invalid obtained status from server. Error: '{}'. Obtained data: '{}'".format( e, current_config)) # if format is not understood, cannot be compared, assume it is ok return True