2023-12-25T01:26:08.121248021Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2023-12-25T01:26:08.121278360Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2023-12-25T01:26:08.121384698Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2023-12-25T01:26:08.121475043Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2023-12-25T01:26:08.121574978Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2023-12-25T01:26:08.121580107Z level=info ts=2023-12-25T01:26:08.121Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2023-12-25T01:26:08.124389854Z level=info ts=2023-12-25T01:26:08.124Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2023-12-25T01:26:08.125133742Z level=info ts=2023-12-25T01:26:08.125Z caller=main.go:824 msg="Starting TSDB ..." 2023-12-25T01:26:08.126115757Z level=info ts=2023-12-25T01:26:08.126Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2023-12-25T01:26:08.133562458Z level=info ts=2023-12-25T01:26:08.133Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-12-25T01:26:08.133574654Z level=info ts=2023-12-25T01:26:08.133Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.797µs 2023-12-25T01:26:08.133579572Z level=info ts=2023-12-25T01:26:08.133Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2023-12-25T01:26:08.134108269Z level=info ts=2023-12-25T01:26:08.134Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-12-25T01:26:08.134203844Z level=info ts=2023-12-25T01:26:08.134Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=138.421µs wal_replay_duration=394.084µs total_replay_duration=555.167µs 2023-12-25T01:26:08.135908234Z level=info ts=2023-12-25T01:26:08.135Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2023-12-25T01:26:08.135933405Z level=info ts=2023-12-25T01:26:08.135Z caller=main.go:854 msg="TSDB started" 2023-12-25T01:26:08.136002410Z level=info ts=2023-12-25T01:26:08.135Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-25T01:26:08.137225199Z level=info ts=2023-12-25T01:26:08.137Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.285555ms remote_storage=6.58µs web_handler=554ns query_engine=3.75µs scrape=746.225µs scrape_sd=24.996µs notify=33.609µs notify_sd=20.863µs rules=3.693µs 2023-12-25T01:26:08.137236035Z level=info ts=2023-12-25T01:26:08.137Z caller=main.go:796 msg="Server is ready to receive web requests." 2023-12-25T01:31:32.942487015Z level=info ts=2023-12-25T01:31:32.942Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-25T01:31:32.944526095Z level=info ts=2023-12-25T01:31:32.944Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.208078ms remote_storage=5.396µs web_handler=3.561µs query_engine=8.224µs scrape=121.179µs scrape_sd=59.828µs notify=31.96µs notify_sd=11.433µs rules=6.839µs 2023-12-25T04:26:39.178500255Z level=info ts=2023-12-25T04:26:39.177Z caller=compact.go:518 component=tsdb msg="write block" mint=1703467584123 maxt=1703469600000 ulid=01HJFKK44S343DA62T8QYGZNER duration=48.079192ms 2023-12-25T04:26:39.181069786Z level=info ts=2023-12-25T04:26:39.180Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.282341ms 2023-12-25T05:00:24.180316849Z level=info ts=2023-12-25T05:00:24.179Z caller=compact.go:518 component=tsdb msg="write block" mint=1703469609123 maxt=1703476800000 ulid=01HJFNGXP2YG1AJTG1E1TGFG44 duration=49.198902ms 2023-12-25T05:00:24.182448445Z level=info ts=2023-12-25T05:00:24.182Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.559651ms 2023-12-25T05:05:56.917487525Z level=info ts=2023-12-25T05:05:56.917Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-25T05:05:56.920117668Z level=info ts=2023-12-25T05:05:56.919Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.903195ms remote_storage=16.616µs web_handler=9.301µs query_engine=10.438µs scrape=325.968µs scrape_sd=357.918µs notify=88.331µs notify_sd=23.077µs rules=18.422µs 2023-12-25T05:06:21.906268307Z level=info ts=2023-12-25T05:06:21.904Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-25T05:06:21.906292783Z level=info ts=2023-12-25T05:06:21.905Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=880.085µs remote_storage=2.45µs web_handler=478ns query_engine=1.234µs scrape=315.404µs scrape_sd=85.367µs notify=13.238µs notify_sd=28.122µs rules=48.469µs 2023-12-25T01:26:30.379572731Z Connected to MongoDB! 2023-12-25T01:26:30.379626560Z Refreshing prometheus config file for first time 2023-12-25T01:27:00.732564737Z Error in first configuration attempt! 2023-12-25T01:27:00.732591187Z mongodb-k8s:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]> 2023-12-25T01:27:30.766611030Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:27:30.766673957Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:28:05.801444341Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:28:05.801504634Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:28:40.830098596Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:28:40.830212883Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:29:15.862642430Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:29:15.862699803Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:29:50.896500130Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:29:50.896560059Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:30:25.926766635Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:30:25.926826886Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:31:00.965300163Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-25T01:31:00.965472732Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-25T01:31:31.614297317Z Listening to changes in prometheus jobs collection 2023-12-25T01:31:32.924115539Z Change detected, updating prometheus config 2023-12-25T01:31:32.929535587Z Jobs detected : 1 2023-12-25T01:31:32.934942782Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-25T01:31:32.936778984Z New config written, updating prometheus 2023-12-25T01:31:37.973752156Z Prometheus config update successful 2023-12-25T01:31:37.976459052Z 2023-12-25T05:05:56.893958637Z Change detected, updating prometheus config 2023-12-25T05:05:56.900248938Z Jobs detected : 2 2023-12-25T05:05:56.907490843Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-25T05:05:56.912501300Z New config written, updating prometheus 2023-12-25T05:06:01.951872434Z Prometheus config update successful 2023-12-25T05:06:01.955790582Z 2023-12-25T05:06:21.906278982Z Change detected, updating prometheus config 2023-12-25T05:06:21.906303064Z Jobs detected : 1 2023-12-25T05:06:21.906307386Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-25T05:06:21.906310651Z New config written, updating prometheus 2023-12-25T05:06:26.924413502Z Prometheus config update successful 2023-12-25T05:06:26.926817453Z