2024-01-26T21:30:41.003243254Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-01-26T21:30:41.003289248Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-01-26T21:30:41.003294102Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-01-26T21:30:41.003297561Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-01-26T21:30:41.003300660Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-01-26T21:30:41.003303803Z level=info ts=2024-01-26T21:30:41.002Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-01-26T21:30:41.004917031Z level=info ts=2024-01-26T21:30:41.004Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-01-26T21:30:41.009180482Z level=info ts=2024-01-26T21:30:41.009Z caller=main.go:824 msg="Starting TSDB ..." 2024-01-26T21:30:41.021051219Z level=info ts=2024-01-26T21:30:41.010Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-01-26T21:30:41.021733648Z level=info ts=2024-01-26T21:30:41.021Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-01-26T21:30:41.022154503Z level=info ts=2024-01-26T21:30:41.021Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=18.125µs 2024-01-26T21:30:41.022166990Z level=info ts=2024-01-26T21:30:41.022Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-01-26T21:30:41.022810134Z level=info ts=2024-01-26T21:30:41.022Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-01-26T21:30:41.023079735Z level=info ts=2024-01-26T21:30:41.022Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=128.911µs wal_replay_duration=511.713µs total_replay_duration=1.08992ms 2024-01-26T21:30:41.025272773Z level=info ts=2024-01-26T21:30:41.025Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-01-26T21:30:41.025383991Z level=info ts=2024-01-26T21:30:41.025Z caller=main.go:854 msg="TSDB started" 2024-01-26T21:30:41.025546965Z level=info ts=2024-01-26T21:30:41.025Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-26T21:30:41.026600730Z level=info ts=2024-01-26T21:30:41.026Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.07625ms remote_storage=2.899µs web_handler=597ns query_engine=5.539µs scrape=411.605µs scrape_sd=40.484µs notify=38.65µs notify_sd=14.967µs rules=4.782µs 2024-01-26T21:30:41.026615226Z level=info ts=2024-01-26T21:30:41.026Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-01-26T21:36:03.380557740Z level=info ts=2024-01-26T21:36:03.380Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-26T21:36:03.381896276Z level=info ts=2024-01-26T21:36:03.381Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.441663ms remote_storage=7.086µs web_handler=717ns query_engine=4.503µs scrape=144.703µs scrape_sd=47.027µs notify=29.355µs notify_sd=10.493µs rules=6.637µs 2024-01-26T21:36:13.425851817Z level=info ts=2024-01-26T21:36:13.425Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-26T21:36:13.426923949Z level=info ts=2024-01-26T21:36:13.426Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.121006ms remote_storage=3.071µs web_handler=707ns query_engine=1.853µs scrape=140.444µs scrape_sd=43.039µs notify=26.986µs notify_sd=23.848µs rules=2.362µs 2024-01-27T00:31:09.178214806Z level=info ts=2024-01-27T00:31:09.177Z caller=compact.go:518 component=tsdb msg="write block" mint=1706304654124 maxt=1706306400000 ulid=01HN457MABRVJPFD0Q7PHZV0AQ duration=46.582429ms 2024-01-27T00:31:09.181531905Z level=info ts=2024-01-27T00:31:09.181Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.508887ms 2024-01-27T01:00:09.194280764Z level=info ts=2024-01-27T01:00:09.193Z caller=compact.go:518 component=tsdb msg="write block" mint=1706306409124 maxt=1706313600000 ulid=01HN46WQHJCBGM693MV7KJJDEY duration=55.176089ms 2024-01-27T01:00:09.196377410Z level=info ts=2024-01-27T01:00:09.196Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.523677ms 2024-01-27T01:07:49.026709774Z level=info ts=2024-01-27T01:07:49.026Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-27T01:07:49.030201589Z level=info ts=2024-01-27T01:07:49.029Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.718252ms remote_storage=5.915µs web_handler=1.064µs query_engine=2.35µs scrape=1.154972ms scrape_sd=110.489µs notify=56.778µs notify_sd=15.116µs rules=10.295µs 2024-01-27T01:08:15.264009366Z level=info ts=2024-01-27T01:08:15.263Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-27T01:08:15.265055024Z level=info ts=2024-01-27T01:08:15.264Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=909.315µs remote_storage=2.644µs web_handler=505ns query_engine=1.357µs scrape=271.265µs scrape_sd=26.791µs notify=15.032µs notify_sd=35.238µs rules=1.616µs 2024-01-26T21:35:02.238450353Z Created MongoClient to connect to MongoDB! 2024-01-26T21:35:02.238662136Z Refreshing prometheus config file for first time 2024-01-26T21:35:32.269627366Z Error in configuration attempt! Number of tries: 1/3 2024-01-26T21:35:32.269673762Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-26T21:35:37.272074012Z Refreshing prometheus config file for first time 2024-01-26T21:36:03.358799990Z Jobs detected : 0 2024-01-26T21:36:03.368336894Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-26T21:36:03.370487765Z alerting: 2024-01-26T21:36:03.370502108Z alertmanagers: 2024-01-26T21:36:03.370505992Z - static_configs: 2024-01-26T21:36:03.370509188Z - targets: null 2024-01-26T21:36:03.370512332Z global: 2024-01-26T21:36:03.370515888Z evaluation_interval: 15s 2024-01-26T21:36:03.370519175Z scrape_interval: 15s 2024-01-26T21:36:03.370522471Z rule_files: null 2024-01-26T21:36:03.370525542Z scrape_configs: 2024-01-26T21:36:03.370528537Z - job_name: mon_exporter 2024-01-26T21:36:03.370531582Z static_configs: 2024-01-26T21:36:03.370534608Z - targets: 2024-01-26T21:36:03.370537681Z - mon:8000 2024-01-26T21:36:03.370758280Z 2024-01-26T21:36:03.373508120Z New config written, updating prometheus 2024-01-26T21:36:08.401538176Z Prometheus config update successful 2024-01-26T21:36:13.403328894Z Refreshing prometheus config file for first time 2024-01-26T21:36:13.406944571Z Jobs detected : 1 2024-01-26T21:36:13.417789213Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-26T21:36:13.420434080Z alerting: 2024-01-26T21:36:13.420459904Z alertmanagers: 2024-01-26T21:36:13.420466936Z - static_configs: 2024-01-26T21:36:13.420473606Z - targets: null 2024-01-26T21:36:13.420479408Z global: 2024-01-26T21:36:13.420485145Z evaluation_interval: 15s 2024-01-26T21:36:13.420490950Z scrape_interval: 15s 2024-01-26T21:36:13.420496789Z rule_files: null 2024-01-26T21:36:13.420502585Z scrape_configs: 2024-01-26T21:36:13.420508322Z - job_name: mon_exporter 2024-01-26T21:36:13.420514122Z static_configs: 2024-01-26T21:36:13.420519934Z - targets: 2024-01-26T21:36:13.420525655Z - mon:8000 2024-01-26T21:36:13.420531528Z 2024-01-26T21:36:13.422246375Z New config written, updating prometheus 2024-01-26T21:36:18.446338997Z Prometheus config update successful 2024-01-26T21:36:23.457288385Z Listening to changes in prometheus jobs collection 2024-01-27T01:07:48.992485345Z Change detected, updating prometheus config 2024-01-27T01:07:48.992635183Z {'_id': {'_data': '8265B45764000000042B022C0100296E5A10042944ACC4EDC14CF084A735CDBE7D9B2746645F6964006465B45764BE9E085671748D960004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1706317668, 4), 'fullDocument': {'_id': ObjectId('65b45764be9e085671748d96'), 'job_name': 'b2f86f7168624e8e8e535c271d292fa9', 'metrics_path': '/snmp', 'nsr_id': '93c1cdcf-42cc-4de7-8581-ae1da0a560c9', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0042524097-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.249.172']}], 'vnfr_id': 'b2f86f7168624e8e8e535c271d292fa9'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65b45764be9e085671748d96')}} 2024-01-27T01:07:48.995990413Z Jobs detected : 2 2024-01-27T01:07:49.005689586Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-27T01:07:49.011187288Z alerting: 2024-01-27T01:07:49.011258146Z alertmanagers: 2024-01-27T01:07:49.011269855Z - static_configs: 2024-01-27T01:07:49.011278527Z - targets: null 2024-01-27T01:07:49.011286659Z global: 2024-01-27T01:07:49.011294930Z evaluation_interval: 15s 2024-01-27T01:07:49.011303608Z scrape_interval: 15s 2024-01-27T01:07:49.011311637Z rule_files: null 2024-01-27T01:07:49.011319560Z scrape_configs: 2024-01-27T01:07:49.011327576Z - job_name: mon_exporter 2024-01-27T01:07:49.011335614Z static_configs: 2024-01-27T01:07:49.011343501Z - targets: 2024-01-27T01:07:49.011351448Z - mon:8000 2024-01-27T01:07:49.011359504Z - job_name: b2f86f7168624e8e8e535c271d292fa9 2024-01-27T01:07:49.011367625Z metrics_path: /snmp 2024-01-27T01:07:49.011375571Z params: 2024-01-27T01:07:49.011383577Z module: 2024-01-27T01:07:49.011405514Z - osm-snmp 2024-01-27T01:07:49.011413440Z relabel_configs: 2024-01-27T01:07:49.011421504Z - source_labels: 2024-01-27T01:07:49.011429288Z - __address__ 2024-01-27T01:07:49.011437406Z target_label: __param_target 2024-01-27T01:07:49.011445366Z - source_labels: 2024-01-27T01:07:49.011453344Z - __param_target 2024-01-27T01:07:49.011461196Z target_label: instance 2024-01-27T01:07:49.011469113Z - replacement: eechart-0042524097-snmpexporter:80 2024-01-27T01:07:49.011477301Z target_label: __address__ 2024-01-27T01:07:49.011485240Z static_configs: 2024-01-27T01:07:49.011525777Z - targets: 2024-01-27T01:07:49.011533622Z - 172.21.249.172 2024-01-27T01:07:49.011540676Z 2024-01-27T01:07:49.016306654Z New config written, updating prometheus 2024-01-27T01:07:54.069453170Z Prometheus config update successful 2024-01-27T01:07:54.072658792Z 2024-01-27T01:08:15.241280999Z Change detected, updating prometheus config 2024-01-27T01:08:15.241337547Z {'_id': {'_data': '8265B4577F000000062B022C0100296E5A10042944ACC4EDC14CF084A735CDBE7D9B2746645F6964006465B45764BE9E085671748D960004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1706317695, 6), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65b45764be9e085671748d96')}} 2024-01-27T01:08:15.244495298Z Jobs detected : 1 2024-01-27T01:08:15.255714597Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-27T01:08:15.258027936Z alerting: 2024-01-27T01:08:15.258089395Z alertmanagers: 2024-01-27T01:08:15.258100496Z - static_configs: 2024-01-27T01:08:15.258109605Z - targets: null 2024-01-27T01:08:15.258118360Z global: 2024-01-27T01:08:15.258127282Z evaluation_interval: 15s 2024-01-27T01:08:15.258136782Z scrape_interval: 15s 2024-01-27T01:08:15.258175454Z rule_files: null 2024-01-27T01:08:15.258184471Z scrape_configs: 2024-01-27T01:08:15.258193087Z - job_name: mon_exporter 2024-01-27T01:08:15.258201926Z static_configs: 2024-01-27T01:08:15.258210687Z - targets: 2024-01-27T01:08:15.258257441Z - mon:8000 2024-01-27T01:08:15.258268076Z 2024-01-27T01:08:15.260058864Z New config written, updating prometheus 2024-01-27T01:08:20.286468589Z Prometheus config update successful 2024-01-27T01:08:20.289034799Z