2024-03-27T01:52:50.572745448Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-03-27T01:52:50.572777772Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-03-27T01:52:50.572788709Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-03-27T01:52:50.572801013Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-03-27T01:52:50.572804305Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-03-27T01:52:50.572806714Z level=info ts=2024-03-27T01:52:50.572Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-27T01:52:50.574679753Z level=info ts=2024-03-27T01:52:50.574Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-03-27T01:52:50.577023481Z level=info ts=2024-03-27T01:52:50.575Z caller=main.go:824 msg="Starting TSDB ..." 2024-03-27T01:52:50.577798048Z level=info ts=2024-03-27T01:52:50.577Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-03-27T01:52:50.580136508Z level=info ts=2024-03-27T01:52:50.579Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-27T01:52:50.580145376Z level=info ts=2024-03-27T01:52:50.580Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.539µs 2024-03-27T01:52:50.580148302Z level=info ts=2024-03-27T01:52:50.580Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-03-27T01:52:50.580699782Z level=info ts=2024-03-27T01:52:50.580Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-03-27T01:52:50.580717528Z level=info ts=2024-03-27T01:52:50.580Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.452µs wal_replay_duration=572.694µs total_replay_duration=625.197µs 2024-03-27T01:52:50.582455990Z level=info ts=2024-03-27T01:52:50.582Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-03-27T01:52:50.582465553Z level=info ts=2024-03-27T01:52:50.582Z caller=main.go:854 msg="TSDB started" 2024-03-27T01:52:50.582468903Z level=info ts=2024-03-27T01:52:50.582Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-27T01:52:50.583226758Z level=info ts=2024-03-27T01:52:50.583Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=721.875µs remote_storage=5.73µs web_handler=393ns query_engine=3.47µs scrape=360.728µs scrape_sd=25.891µs notify=40.141µs notify_sd=9.304µs rules=3.982µs 2024-03-27T01:52:50.583235119Z level=info ts=2024-03-27T01:52:50.583Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-03-27T01:56:58.355635914Z level=info ts=2024-03-27T01:56:58.355Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-27T01:56:58.356983622Z level=info ts=2024-03-27T01:56:58.356Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.270583ms remote_storage=9.358µs web_handler=4.839µs query_engine=8.141µs scrape=152.277µs scrape_sd=75.876µs notify=74.55µs notify_sd=19.109µs rules=6.439µs 2024-03-27T04:53:24.179836865Z level=info ts=2024-03-27T04:53:24.179Z caller=compact.go:518 component=tsdb msg="write block" mint=1711504389124 maxt=1711504800000 ulid=01HSZ42YHFNXQXTPWET1AC886M duration=35.988925ms 2024-03-27T04:53:24.183108087Z level=info ts=2024-03-27T04:53:24.182Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.420154ms 2024-03-27T05:00:24.217263246Z level=info ts=2024-03-27T05:00:24.216Z caller=compact.go:518 component=tsdb msg="write block" mint=1711504809127 maxt=1711512000000 ulid=01HSZ4FRP939EVBS35AX3EX4WS duration=79.133525ms 2024-03-27T05:00:24.220162489Z level=info ts=2024-03-27T05:00:24.219Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.193658ms 2024-03-27T05:23:41.958325822Z level=info ts=2024-03-27T05:23:41.957Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-27T05:23:41.965580192Z level=info ts=2024-03-27T05:23:41.965Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=7.579524ms remote_storage=5.585µs web_handler=12.081µs query_engine=63.623µs scrape=390.413µs scrape_sd=217.911µs notify=118.017µs notify_sd=68.799µs rules=5.673µs 2024-03-27T05:25:00.253144651Z level=info ts=2024-03-27T05:25:00.252Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-27T05:25:00.254360473Z level=info ts=2024-03-27T05:25:00.254Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.266785ms remote_storage=3.461µs web_handler=1.016µs query_engine=2.712µs scrape=353.536µs scrape_sd=50.391µs notify=22.38µs notify_sd=21.687µs rules=2.312µs 2024-03-27T01:53:21.250257445Z Connected to MongoDB! 2024-03-27T01:53:21.250323496Z Refreshing prometheus config file for first time 2024-03-27T01:53:51.290980579Z Error in first configuration attempt! 2024-03-27T01:53:51.291004490Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:54:21.315160065Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:54:21.315199775Z Detected failure while listening to prometheus jobs collection, retrying... 2024-03-27T01:54:56.347140780Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:54:56.347173886Z Detected failure while listening to prometheus jobs collection, retrying... 2024-03-27T01:55:31.369103873Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:55:31.369139062Z Detected failure while listening to prometheus jobs collection, retrying... 2024-03-27T01:56:06.412681553Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:56:06.412735305Z Detected failure while listening to prometheus jobs collection, retrying... 2024-03-27T01:56:41.450944851Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-03-27T01:56:41.451482222Z Detected failure while listening to prometheus jobs collection, retrying... 2024-03-27T01:56:57.023888414Z Listening to changes in prometheus jobs collection 2024-03-27T01:56:58.341980322Z Change detected, updating prometheus config 2024-03-27T01:56:58.345279533Z Jobs detected : 1 2024-03-27T01:56:58.349107734Z Writing new config file to /etc/prometheus/prometheus.yml 2024-03-27T01:56:58.350322878Z New config written, updating prometheus 2024-03-27T01:57:03.370819184Z Prometheus config update successful 2024-03-27T01:57:03.373053423Z 2024-03-27T05:23:41.935284910Z Change detected, updating prometheus config 2024-03-27T05:23:41.939796394Z Jobs detected : 2 2024-03-27T05:23:41.942444805Z Writing new config file to /etc/prometheus/prometheus.yml 2024-03-27T05:23:41.948701760Z New config written, updating prometheus 2024-03-27T05:23:46.999475807Z Prometheus config update successful 2024-03-27T05:23:47.002900797Z 2024-03-27T05:25:00.234222300Z Change detected, updating prometheus config 2024-03-27T05:25:00.240211762Z Jobs detected : 1 2024-03-27T05:25:00.247550891Z Writing new config file to /etc/prometheus/prometheus.yml 2024-03-27T05:25:00.249466677Z New config written, updating prometheus 2024-03-27T05:25:05.274216992Z Prometheus config update successful 2024-03-27T05:25:05.276211225Z