2023-12-30T01:25:02.777132070Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2023-12-30T01:25:02.777164058Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2023-12-30T01:25:02.777168540Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2023-12-30T01:25:02.777171648Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2023-12-30T01:25:02.777174694Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2023-12-30T01:25:02.777177542Z level=info ts=2023-12-30T01:25:02.776Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2023-12-30T01:25:02.782099157Z level=info ts=2023-12-30T01:25:02.782Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2023-12-30T01:25:02.784727244Z level=info ts=2023-12-30T01:25:02.784Z caller=main.go:824 msg="Starting TSDB ..." 2023-12-30T01:25:02.787210370Z level=info ts=2023-12-30T01:25:02.787Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2023-12-30T01:25:02.794676671Z level=info ts=2023-12-30T01:25:02.794Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-12-30T01:25:02.794792149Z level=info ts=2023-12-30T01:25:02.794Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=9.649µs 2023-12-30T01:25:02.794806451Z level=info ts=2023-12-30T01:25:02.794Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2023-12-30T01:25:02.795452795Z level=info ts=2023-12-30T01:25:02.795Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-12-30T01:25:02.795510489Z level=info ts=2023-12-30T01:25:02.795Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.985µs wal_replay_duration=509.456µs total_replay_duration=764.628µs 2023-12-30T01:25:02.797435684Z level=info ts=2023-12-30T01:25:02.797Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2023-12-30T01:25:02.797464226Z level=info ts=2023-12-30T01:25:02.797Z caller=main.go:854 msg="TSDB started" 2023-12-30T01:25:02.797518263Z level=info ts=2023-12-30T01:25:02.797Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-30T01:25:02.798294590Z level=info ts=2023-12-30T01:25:02.798Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=684.108µs remote_storage=9.146µs web_handler=364ns query_engine=3.447µs scrape=294.822µs scrape_sd=23.404µs notify=51.462µs notify_sd=10.296µs rules=3.893µs 2023-12-30T01:25:02.798321793Z level=info ts=2023-12-30T01:25:02.798Z caller=main.go:796 msg="Server is ready to receive web requests." 2023-12-30T01:29:57.336207338Z level=info ts=2023-12-30T01:29:57.336Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-30T01:29:57.337158238Z level=info ts=2023-12-30T01:29:57.337Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.051718ms remote_storage=6.17µs web_handler=3.808µs query_engine=6.683µs scrape=124.576µs scrape_sd=34.795µs notify=31.833µs notify_sd=9.872µs rules=5.636µs 2023-12-30T04:25:24.178365773Z level=info ts=2023-12-30T04:25:24.178Z caller=compact.go:518 component=tsdb msg="write block" mint=1703899509125 maxt=1703901600000 ulid=01HJWFGDX4815QNDJ50QEB55G0 duration=45.089315ms 2023-12-30T04:25:24.181631616Z level=info ts=2023-12-30T04:25:24.181Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.020473ms 2023-12-30T05:00:24.195491160Z level=info ts=2023-12-30T05:00:24.195Z caller=compact.go:518 component=tsdb msg="write block" mint=1703901609125 maxt=1703908800000 ulid=01HJWHGGP8SN265W8QGZCSV7FM duration=58.970397ms 2023-12-30T05:00:24.198014760Z level=info ts=2023-12-30T05:00:24.197Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.09347ms 2023-12-30T05:06:13.295834937Z level=info ts=2023-12-30T05:06:13.295Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-30T05:06:13.298282513Z level=info ts=2023-12-30T05:06:13.298Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.596174ms remote_storage=6.677µs web_handler=7.448µs query_engine=6.598µs scrape=221.823µs scrape_sd=698.584µs notify=49.828µs notify_sd=14.587µs rules=15.514µs 2023-12-30T05:06:38.130615631Z level=info ts=2023-12-30T05:06:38.130Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-12-30T05:06:38.131361941Z level=info ts=2023-12-30T05:06:38.131Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=784.766µs remote_storage=2.593µs web_handler=614ns query_engine=1.255µs scrape=266.026µs scrape_sd=27.322µs notify=21.411µs notify_sd=27.52µs rules=1.288µs 2023-12-30T01:25:25.979181144Z Connected to MongoDB! 2023-12-30T01:25:25.979220352Z Refreshing prometheus config file for first time 2023-12-30T01:25:56.341815634Z Error in first configuration attempt! 2023-12-30T01:25:56.341851083Z mongodb-k8s:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]> 2023-12-30T01:26:26.383387452Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:26:26.383437553Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:27:01.398050033Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:27:01.398110304Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:27:36.432008342Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:27:36.432069634Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:28:11.453372538Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:28:11.453488408Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:28:46.499813715Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:28:46.499879372Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:29:21.519858807Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2023-12-30T01:29:21.519915891Z Detected failure while listening to prometheus jobs collection, retrying... 2023-12-30T01:29:55.906948199Z Listening to changes in prometheus jobs collection 2023-12-30T01:29:57.316363941Z Change detected, updating prometheus config 2023-12-30T01:29:57.319905476Z Jobs detected : 1 2023-12-30T01:29:57.329301915Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-30T01:29:57.329318427Z New config written, updating prometheus 2023-12-30T01:30:02.357414559Z Prometheus config update successful 2023-12-30T01:30:02.359777407Z 2023-12-30T05:06:13.268259271Z Change detected, updating prometheus config 2023-12-30T05:06:13.272751621Z Jobs detected : 2 2023-12-30T05:06:13.279991547Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-30T05:06:13.286917001Z New config written, updating prometheus 2023-12-30T05:06:18.331301923Z Prometheus config update successful 2023-12-30T05:06:18.334749934Z 2023-12-30T05:06:38.108729592Z Change detected, updating prometheus config 2023-12-30T05:06:38.110317238Z Jobs detected : 1 2023-12-30T05:06:38.122864273Z Writing new config file to /etc/prometheus/prometheus.yml 2023-12-30T05:06:38.125487497Z New config written, updating prometheus 2023-12-30T05:06:43.149416214Z Prometheus config update successful 2023-12-30T05:06:43.151295628Z