2024-01-13T21:34:33.908337585Z Created MongoClient to connect to MongoDB! 2024-01-13T21:34:33.916295914Z Refreshing prometheus config file for first time 2024-01-13T21:35:03.946177408Z Error in configuration attempt! Number of tries: 1/3 2024-01-13T21:35:03.946239234Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-13T21:35:08.949624199Z Refreshing prometheus config file for first time 2024-01-13T21:35:28.835726039Z Jobs detected : 0 2024-01-13T21:35:28.841295241Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-13T21:35:28.849919569Z alerting: 2024-01-13T21:35:28.849936735Z alertmanagers: 2024-01-13T21:35:28.849940823Z - static_configs: 2024-01-13T21:35:28.849944494Z - targets: null 2024-01-13T21:35:28.849947921Z global: 2024-01-13T21:35:28.849951476Z evaluation_interval: 15s 2024-01-13T21:35:28.849954893Z scrape_interval: 15s 2024-01-13T21:35:28.849958697Z rule_files: null 2024-01-13T21:35:28.849962110Z scrape_configs: 2024-01-13T21:35:28.849965515Z - job_name: mon_exporter 2024-01-13T21:35:28.849968931Z static_configs: 2024-01-13T21:35:28.849972423Z - targets: 2024-01-13T21:35:28.849975824Z - mon:8000 2024-01-13T21:35:28.849979202Z 2024-01-13T21:35:28.852660904Z New config written, updating prometheus 2024-01-13T21:35:33.888157690Z Prometheus config update successful 2024-01-13T21:35:38.890712771Z Refreshing prometheus config file for first time 2024-01-13T21:35:38.896730063Z Jobs detected : 1 2024-01-13T21:35:38.906499516Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-13T21:35:38.908794925Z alerting: 2024-01-13T21:35:38.908831624Z alertmanagers: 2024-01-13T21:35:38.908842713Z - static_configs: 2024-01-13T21:35:38.908854197Z - targets: null 2024-01-13T21:35:38.908867934Z global: 2024-01-13T21:35:38.908882157Z evaluation_interval: 15s 2024-01-13T21:35:38.908900158Z scrape_interval: 15s 2024-01-13T21:35:38.908916461Z rule_files: null 2024-01-13T21:35:38.908932664Z scrape_configs: 2024-01-13T21:35:38.908948591Z - job_name: mon_exporter 2024-01-13T21:35:38.908960726Z static_configs: 2024-01-13T21:35:38.908975026Z - targets: 2024-01-13T21:35:38.908991512Z - mon:8000 2024-01-13T21:35:38.909007600Z 2024-01-13T21:35:38.910930692Z New config written, updating prometheus 2024-01-13T21:35:43.938034038Z Prometheus config update successful 2024-01-13T21:35:48.951518172Z Listening to changes in prometheus jobs collection 2024-01-14T01:15:10.251575038Z Change detected, updating prometheus config 2024-01-14T01:15:10.252087214Z {'_id': {'_data': '8265A3359E000000042B022C0100296E5A1004556C8BEE57A84F9DBDBC07E750A1934246645F6964006465A3359E42F561F5E1D3D8CF0004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1705194910, 4), 'fullDocument': {'_id': ObjectId('65a3359e42f561f5e1d3d8cf'), 'job_name': '82ed3145be924e4ba0dedb44ef2e0773', 'metrics_path': '/snmp', 'nsr_id': '068102f8-f65a-451a-8623-713b97757715', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0072438053-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.248.132']}], 'vnfr_id': '82ed3145be924e4ba0dedb44ef2e0773'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65a3359e42f561f5e1d3d8cf')}} 2024-01-14T01:15:10.259676771Z Jobs detected : 2 2024-01-14T01:15:10.269823798Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-14T01:15:10.274016721Z alerting: 2024-01-14T01:15:10.274034332Z alertmanagers: 2024-01-14T01:15:10.274038969Z - static_configs: 2024-01-14T01:15:10.274042942Z - targets: null 2024-01-14T01:15:10.274046695Z global: 2024-01-14T01:15:10.274050436Z evaluation_interval: 15s 2024-01-14T01:15:10.274054396Z scrape_interval: 15s 2024-01-14T01:15:10.274058147Z rule_files: null 2024-01-14T01:15:10.274062132Z scrape_configs: 2024-01-14T01:15:10.274065873Z - job_name: mon_exporter 2024-01-14T01:15:10.274069682Z static_configs: 2024-01-14T01:15:10.274073449Z - targets: 2024-01-14T01:15:10.274077141Z - mon:8000 2024-01-14T01:15:10.274080889Z - job_name: 82ed3145be924e4ba0dedb44ef2e0773 2024-01-14T01:15:10.274084768Z metrics_path: /snmp 2024-01-14T01:15:10.274088502Z params: 2024-01-14T01:15:10.274093081Z module: 2024-01-14T01:15:10.274096859Z - osm-snmp 2024-01-14T01:15:10.274100535Z relabel_configs: 2024-01-14T01:15:10.274104367Z - source_labels: 2024-01-14T01:15:10.274108121Z - __address__ 2024-01-14T01:15:10.274111859Z target_label: __param_target 2024-01-14T01:15:10.274115646Z - source_labels: 2024-01-14T01:15:10.274119356Z - __param_target 2024-01-14T01:15:10.274123076Z target_label: instance 2024-01-14T01:15:10.274126819Z - replacement: eechart-0072438053-snmpexporter:80 2024-01-14T01:15:10.274130649Z target_label: __address__ 2024-01-14T01:15:10.274134379Z static_configs: 2024-01-14T01:15:10.274151599Z - targets: 2024-01-14T01:15:10.274155156Z - 172.21.248.132 2024-01-14T01:15:10.274158674Z 2024-01-14T01:15:10.277834336Z New config written, updating prometheus 2024-01-14T01:15:15.321659947Z Prometheus config update successful 2024-01-14T01:15:15.325263427Z 2024-01-14T01:15:33.710682455Z Change detected, updating prometheus config 2024-01-14T01:15:33.710766958Z {'_id': {'_data': '8265A335B5000000082B022C0100296E5A1004556C8BEE57A84F9DBDBC07E750A1934246645F6964006465A3359E42F561F5E1D3D8CF0004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1705194933, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65a3359e42f561f5e1d3d8cf')}} 2024-01-14T01:15:33.710781302Z Jobs detected : 1 2024-01-14T01:15:33.715081850Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-14T01:15:33.717527572Z alerting: 2024-01-14T01:15:33.717578770Z alertmanagers: 2024-01-14T01:15:33.717589177Z - static_configs: 2024-01-14T01:15:33.717595605Z - targets: null 2024-01-14T01:15:33.717602066Z global: 2024-01-14T01:15:33.717608097Z evaluation_interval: 15s 2024-01-14T01:15:33.717629634Z scrape_interval: 15s 2024-01-14T01:15:33.717636055Z rule_files: null 2024-01-14T01:15:33.717642047Z scrape_configs: 2024-01-14T01:15:33.717648062Z - job_name: mon_exporter 2024-01-14T01:15:33.717654154Z static_configs: 2024-01-14T01:15:33.717660192Z - targets: 2024-01-14T01:15:33.717666267Z - mon:8000 2024-01-14T01:15:33.717690729Z 2024-01-14T01:15:33.719337289Z New config written, updating prometheus 2024-01-14T01:15:38.747693694Z Prometheus config update successful 2024-01-14T01:15:38.753253736Z 2024-01-13T21:30:15.643001953Z level=info ts=2024-01-13T21:30:15.642Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-01-13T21:30:15.643149948Z level=info ts=2024-01-13T21:30:15.642Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-01-13T21:30:15.643173630Z level=info ts=2024-01-13T21:30:15.642Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-01-13T21:30:15.643177881Z level=info ts=2024-01-13T21:30:15.642Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-01-13T21:30:15.643201418Z level=info ts=2024-01-13T21:30:15.643Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-01-13T21:30:15.643298225Z level=info ts=2024-01-13T21:30:15.643Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-01-13T21:30:15.645516902Z level=info ts=2024-01-13T21:30:15.645Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-01-13T21:30:15.646229791Z level=info ts=2024-01-13T21:30:15.646Z caller=main.go:824 msg="Starting TSDB ..." 2024-01-13T21:30:15.666405709Z level=info ts=2024-01-13T21:30:15.665Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-01-13T21:30:15.700292121Z level=info ts=2024-01-13T21:30:15.700Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-01-13T21:30:15.700412415Z level=info ts=2024-01-13T21:30:15.700Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=8.011µs 2024-01-13T21:30:15.705368222Z level=info ts=2024-01-13T21:30:15.700Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-01-13T21:30:15.706656557Z level=info ts=2024-01-13T21:30:15.706Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-01-13T21:30:15.706728421Z level=info ts=2024-01-13T21:30:15.706Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.96µs wal_replay_duration=6.161819ms total_replay_duration=6.341914ms 2024-01-13T21:30:15.710497307Z level=info ts=2024-01-13T21:30:15.709Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-01-13T21:30:15.710512521Z level=info ts=2024-01-13T21:30:15.709Z caller=main.go:854 msg="TSDB started" 2024-01-13T21:30:15.710516430Z level=info ts=2024-01-13T21:30:15.709Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-13T21:30:15.713531700Z level=info ts=2024-01-13T21:30:15.713Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.487159ms remote_storage=2.453µs web_handler=823ns query_engine=6.043µs scrape=1.569978ms scrape_sd=58.994µs notify=42.074µs notify_sd=254.323µs rules=107.46µs 2024-01-13T21:30:15.713674508Z level=info ts=2024-01-13T21:30:15.713Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-01-13T21:35:28.859907003Z level=info ts=2024-01-13T21:35:28.859Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-13T21:35:28.863143314Z level=info ts=2024-01-13T21:35:28.863Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.340433ms remote_storage=8.682µs web_handler=977ns query_engine=4.93µs scrape=156.54µs scrape_sd=84.145µs notify=60.495µs notify_sd=24.664µs rules=25.916µs 2024-01-13T21:35:38.915798041Z level=info ts=2024-01-13T21:35:38.915Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-13T21:35:38.916710071Z level=info ts=2024-01-13T21:35:38.916Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.137744ms remote_storage=3.426µs web_handler=797ns query_engine=2.025µs scrape=155.117µs scrape_sd=44.384µs notify=46.578µs notify_sd=56.715µs rules=3.364µs 2024-01-14T00:30:39.171817664Z level=info ts=2024-01-14T00:30:39.171Z caller=compact.go:518 component=tsdb msg="write block" mint=1705181424126 maxt=1705183200000 ulid=01HM2P1C0W3TTGXN4Q3746GWJC duration=38.757184ms 2024-01-14T00:30:39.174664873Z level=info ts=2024-01-14T00:30:39.174Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.324834ms 2024-01-14T01:00:09.200805598Z level=info ts=2024-01-14T01:00:09.200Z caller=compact.go:518 component=tsdb msg="write block" mint=1705183209126 maxt=1705190400000 ulid=01HM2QQCHFF4JC1G5G7VGWE9CZ duration=64.749897ms 2024-01-14T01:00:09.202894444Z level=info ts=2024-01-14T01:00:09.202Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.622424ms 2024-01-14T01:15:10.285246507Z level=info ts=2024-01-14T01:15:10.284Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-14T01:15:10.288423648Z level=info ts=2024-01-14T01:15:10.288Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.567139ms remote_storage=9.176µs web_handler=2.234µs query_engine=3.115µs scrape=227.618µs scrape_sd=117.481µs notify=62.921µs notify_sd=24.081µs rules=16.164µs 2024-01-14T01:15:33.723441546Z level=info ts=2024-01-14T01:15:33.723Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-14T01:15:33.724393562Z level=info ts=2024-01-14T01:15:33.724Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.025023ms remote_storage=2.961µs web_handler=775ns query_engine=1.584µs scrape=313.517µs scrape_sd=37.441µs notify=17.639µs notify_sd=14.802µs rules=9.535µs