2024-01-28T01:24:53.393239164Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-01-28T01:24:53.393274538Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-01-28T01:24:53.393278902Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-01-28T01:24:53.393291867Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-01-28T01:24:53.393295402Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-01-28T01:24:53.393298434Z level=info ts=2024-01-28T01:24:53.386Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-01-28T01:24:53.437254081Z level=info ts=2024-01-28T01:24:53.396Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-01-28T01:24:53.437277134Z level=info ts=2024-01-28T01:24:53.398Z caller=main.go:824 msg="Starting TSDB ..." 2024-01-28T01:24:53.437281450Z level=info ts=2024-01-28T01:24:53.399Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-01-28T01:24:53.437285200Z level=info ts=2024-01-28T01:24:53.414Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-01-28T01:24:53.437288740Z level=info ts=2024-01-28T01:24:53.414Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=8.745µs 2024-01-28T01:24:53.437293056Z level=info ts=2024-01-28T01:24:53.414Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-01-28T01:24:53.437296365Z level=info ts=2024-01-28T01:24:53.415Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-01-28T01:24:53.437299713Z level=info ts=2024-01-28T01:24:53.415Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.798µs wal_replay_duration=193.006µs total_replay_duration=234.317µs 2024-01-28T01:24:53.437303367Z level=info ts=2024-01-28T01:24:53.416Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-01-28T01:24:53.437306617Z level=info ts=2024-01-28T01:24:53.416Z caller=main.go:854 msg="TSDB started" 2024-01-28T01:24:53.437328294Z level=info ts=2024-01-28T01:24:53.416Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-28T01:24:53.439492560Z level=info ts=2024-01-28T01:24:53.438Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.652723ms remote_storage=5.538µs web_handler=563ns query_engine=4.451µs scrape=9.953823ms scrape_sd=41.172µs notify=48.237µs notify_sd=11.098µs rules=37.048µs 2024-01-28T01:24:53.439510781Z level=info ts=2024-01-28T01:24:53.438Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-01-28T01:29:46.554707695Z level=info ts=2024-01-28T01:29:46.554Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-28T01:29:46.556341233Z level=info ts=2024-01-28T01:29:46.556Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.678893ms remote_storage=4.211µs web_handler=4.268µs query_engine=5.816µs scrape=125.1µs scrape_sd=35.624µs notify=27.574µs notify_sd=9.306µs rules=5.714µs 2024-01-28T04:25:24.181167467Z level=info ts=2024-01-28T04:25:24.180Z caller=compact.go:518 component=tsdb msg="write block" mint=1706405109123 maxt=1706407200000 ulid=01HN7518XCBFZYEKVWMMSN7NK5 duration=39.960692ms 2024-01-28T04:25:24.184441669Z level=info ts=2024-01-28T04:25:24.184Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.394801ms 2024-01-28T04:58:11.672905847Z level=info ts=2024-01-28T04:58:11.672Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-28T04:58:11.675276830Z level=info ts=2024-01-28T04:58:11.675Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.83313ms remote_storage=4.853µs web_handler=9.894µs query_engine=18.862µs scrape=279.474µs scrape_sd=163.109µs notify=84.026µs notify_sd=22.679µs rules=16.684µs 2024-01-28T04:58:36.565717396Z level=info ts=2024-01-28T04:58:36.565Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-28T04:58:36.567138581Z level=info ts=2024-01-28T04:58:36.566Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=900.486µs remote_storage=2.712µs web_handler=599ns query_engine=2.138µs scrape=320.788µs scrape_sd=56.453µs notify=30.765µs notify_sd=21.034µs rules=2.141µs 2024-01-28T05:00:24.185688868Z level=info ts=2024-01-28T05:00:24.185Z caller=compact.go:518 component=tsdb msg="write block" mint=1706407209123 maxt=1706414400000 ulid=01HN771BP536YN0N3DQ0HEZDCP duration=51.563887ms 2024-01-28T05:00:24.188113905Z level=info ts=2024-01-28T05:00:24.187Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.959214ms 2024-01-28T01:25:17.466310146Z Connected to MongoDB! 2024-01-28T01:25:17.466391854Z Refreshing prometheus config file for first time 2024-01-28T01:25:47.890159795Z Error in first configuration attempt! 2024-01-28T01:25:47.890235916Z mongodb-k8s:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]> 2024-01-28T01:26:18.029865752Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:26:18.029920515Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:26:53.060399638Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:26:53.060486709Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:27:28.082651637Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:27:28.082719277Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:28:03.127093712Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:28:03.128335629Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:28:38.163763102Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:28:38.163861523Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:29:13.190475652Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-28T01:29:13.190538857Z Detected failure while listening to prometheus jobs collection, retrying... 2024-01-28T01:29:45.204710035Z Listening to changes in prometheus jobs collection 2024-01-28T01:29:46.521267639Z Change detected, updating prometheus config 2024-01-28T01:29:46.534271401Z Jobs detected : 1 2024-01-28T01:29:46.543190255Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-28T01:29:46.546855047Z New config written, updating prometheus 2024-01-28T01:29:51.582080863Z Prometheus config update successful 2024-01-28T01:29:51.585421189Z 2024-01-28T04:58:11.649950241Z Change detected, updating prometheus config 2024-01-28T04:58:11.654678604Z Jobs detected : 2 2024-01-28T04:58:11.662117208Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-28T04:58:11.666518452Z New config written, updating prometheus 2024-01-28T04:58:16.708120744Z Prometheus config update successful 2024-01-28T04:58:16.711758146Z 2024-01-28T04:58:36.544116119Z Change detected, updating prometheus config 2024-01-28T04:58:36.545126054Z Jobs detected : 1 2024-01-28T04:58:36.557752157Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-28T04:58:36.559993041Z New config written, updating prometheus 2024-01-28T04:58:41.588264945Z Prometheus config update successful 2024-01-28T04:58:41.590351406Z