2024-01-31T21:26:43.530558597Z level=info ts=2024-01-31T21:26:43.529Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-01-31T21:26:43.550088584Z level=info ts=2024-01-31T21:26:43.529Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-01-31T21:26:43.550106574Z level=info ts=2024-01-31T21:26:43.529Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-01-31T21:26:43.550109982Z level=info ts=2024-01-31T21:26:43.532Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-01-31T21:26:43.550112973Z level=info ts=2024-01-31T21:26:43.532Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-01-31T21:26:43.550115677Z level=info ts=2024-01-31T21:26:43.532Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-01-31T21:26:43.550118319Z level=info ts=2024-01-31T21:26:43.537Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-01-31T21:26:43.550120953Z level=info ts=2024-01-31T21:26:43.539Z caller=main.go:824 msg="Starting TSDB ..." 2024-01-31T21:26:43.550123557Z level=info ts=2024-01-31T21:26:43.543Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-01-31T21:26:43.550126208Z level=info ts=2024-01-31T21:26:43.543Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-01-31T21:26:43.550129120Z level=info ts=2024-01-31T21:26:43.543Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=8.062µs 2024-01-31T21:26:43.550132654Z level=info ts=2024-01-31T21:26:43.543Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-01-31T21:26:43.550135178Z level=info ts=2024-01-31T21:26:43.544Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-01-31T21:26:43.550137772Z level=info ts=2024-01-31T21:26:43.544Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.669µs wal_replay_duration=343.737µs total_replay_duration=384.372µs 2024-01-31T21:26:43.550140671Z level=info ts=2024-01-31T21:26:43.545Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-01-31T21:26:43.550143255Z level=info ts=2024-01-31T21:26:43.545Z caller=main.go:854 msg="TSDB started" 2024-01-31T21:26:43.550145934Z level=info ts=2024-01-31T21:26:43.545Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-31T21:26:43.550149538Z level=info ts=2024-01-31T21:26:43.546Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=658.016µs remote_storage=9.573µs web_handler=286ns query_engine=2.602µs scrape=200.05µs scrape_sd=25.769µs notify=25.839µs notify_sd=7.835µs rules=3.323µs 2024-01-31T21:26:43.550156825Z level=info ts=2024-01-31T21:26:43.546Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-01-31T21:32:24.893032368Z level=info ts=2024-01-31T21:32:24.892Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-31T21:32:24.895303914Z level=info ts=2024-01-31T21:32:24.895Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.453409ms remote_storage=9.697µs web_handler=14.124µs query_engine=5.053µs scrape=131.959µs scrape_sd=54.356µs notify=50.891µs notify_sd=42.332µs rules=16.014µs 2024-02-01T00:27:09.177516093Z level=info ts=2024-02-01T00:27:09.177Z caller=compact.go:518 component=tsdb msg="write block" mint=1706736414123 maxt=1706738400000 ulid=01HNH0ZWYBQATD0W9BN72S7PGJ duration=45.619875ms 2024-02-01T00:27:09.178853817Z level=info ts=2024-02-01T00:27:09.178Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=967.978µs 2024-02-01T01:00:24.183413468Z level=info ts=2024-02-01T01:00:24.183Z caller=compact.go:518 component=tsdb msg="write block" mint=1706738409123 maxt=1706745600000 ulid=01HNH2WS61CY5ZG57V66S4F4K0 duration=54.012925ms 2024-02-01T01:00:24.185028960Z level=info ts=2024-02-01T01:00:24.184Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.17174ms 2024-02-01T01:07:51.178852643Z level=info ts=2024-02-01T01:07:51.178Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-01T01:07:51.181362382Z level=info ts=2024-02-01T01:07:51.181Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.77753ms remote_storage=4.916µs web_handler=10.615µs query_engine=12.137µs scrape=301.044µs scrape_sd=153.629µs notify=77.805µs notify_sd=46.143µs rules=15.723µs 2024-02-01T01:08:16.234165428Z level=info ts=2024-02-01T01:08:16.234Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-01T01:08:16.234858141Z level=info ts=2024-02-01T01:08:16.234Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=654.984µs remote_storage=2.411µs web_handler=549ns query_engine=1.201µs scrape=250.858µs scrape_sd=30.275µs notify=56.755µs notify_sd=14.009µs rules=1.41µs 2024-01-31T21:30:55.990553190Z Created MongoClient to connect to MongoDB! 2024-01-31T21:30:55.990687339Z Refreshing prometheus config file for first time 2024-01-31T21:31:26.024420417Z Error in configuration attempt! Number of tries: 1/3 2024-01-31T21:31:26.024467711Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-31T21:31:31.029655204Z Refreshing prometheus config file for first time 2024-01-31T21:32:01.057660981Z Error in configuration attempt! Number of tries: 2/3 2024-01-31T21:32:01.057707614Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-31T21:32:06.062823717Z Refreshing prometheus config file for first time 2024-01-31T21:32:24.861337329Z Jobs detected : 0 2024-01-31T21:32:24.876221008Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-31T21:32:24.881608457Z alerting: 2024-01-31T21:32:24.881648457Z alertmanagers: 2024-01-31T21:32:24.881654706Z - static_configs: 2024-01-31T21:32:24.881659079Z - targets: null 2024-01-31T21:32:24.881663660Z global: 2024-01-31T21:32:24.881667968Z evaluation_interval: 15s 2024-01-31T21:32:24.881672181Z scrape_interval: 15s 2024-01-31T21:32:24.881676289Z rule_files: null 2024-01-31T21:32:24.881680369Z scrape_configs: 2024-01-31T21:32:24.881684435Z - job_name: mon_exporter 2024-01-31T21:32:24.881688590Z static_configs: 2024-01-31T21:32:24.881692714Z - targets: 2024-01-31T21:32:24.881696933Z - mon:8000 2024-01-31T21:32:24.881701056Z 2024-01-31T21:32:24.884498356Z New config written, updating prometheus 2024-01-31T21:32:29.924698258Z Prometheus config update successful 2024-01-31T21:32:34.931612481Z Listening to changes in prometheus jobs collection 2024-02-01T01:07:51.151031756Z Change detected, updating prometheus config 2024-02-01T01:07:51.151271348Z {'_id': {'_data': '8265BAEEE7000000042B022C0100296E5A10042C93C5C69BAC4D03B9D4678B057070DB46645F6964006465BAEEE7EE5B0F65E57292AE0004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1706749671, 4), 'fullDocument': {'_id': ObjectId('65baeee7ee5b0f65e57292ae'), 'job_name': '8b67eb5d71e64440a38f9ae6669c0a3b', 'metrics_path': '/snmp', 'nsr_id': '3b11ca93-f73d-454e-8d7f-70de9b88303a', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0062924674-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.249.7']}], 'vnfr_id': '8b67eb5d71e64440a38f9ae6669c0a3b'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65baeee7ee5b0f65e57292ae')}} 2024-02-01T01:07:51.154234961Z Jobs detected : 2 2024-02-01T01:07:51.164302888Z Writing new config file to /etc/prometheus/prometheus.yml 2024-02-01T01:07:51.168016285Z alerting: 2024-02-01T01:07:51.168031821Z alertmanagers: 2024-02-01T01:07:51.168035722Z - static_configs: 2024-02-01T01:07:51.168039011Z - targets: null 2024-02-01T01:07:51.168041971Z global: 2024-02-01T01:07:51.168044992Z evaluation_interval: 15s 2024-02-01T01:07:51.168048057Z scrape_interval: 15s 2024-02-01T01:07:51.168051310Z rule_files: null 2024-02-01T01:07:51.168054355Z scrape_configs: 2024-02-01T01:07:51.168057345Z - job_name: mon_exporter 2024-02-01T01:07:51.168060269Z static_configs: 2024-02-01T01:07:51.168063194Z - targets: 2024-02-01T01:07:51.168066102Z - mon:8000 2024-02-01T01:07:51.168069142Z - job_name: 8b67eb5d71e64440a38f9ae6669c0a3b 2024-02-01T01:07:51.168072136Z metrics_path: /snmp 2024-02-01T01:07:51.168075108Z params: 2024-02-01T01:07:51.168077975Z module: 2024-02-01T01:07:51.168080879Z - osm-snmp 2024-02-01T01:07:51.168083780Z relabel_configs: 2024-02-01T01:07:51.168086795Z - source_labels: 2024-02-01T01:07:51.168089672Z - __address__ 2024-02-01T01:07:51.168092604Z target_label: __param_target 2024-02-01T01:07:51.168095555Z - source_labels: 2024-02-01T01:07:51.168098449Z - __param_target 2024-02-01T01:07:51.168101343Z target_label: instance 2024-02-01T01:07:51.168104306Z - replacement: eechart-0062924674-snmpexporter:80 2024-02-01T01:07:51.168107283Z target_label: __address__ 2024-02-01T01:07:51.168110219Z static_configs: 2024-02-01T01:07:51.168113120Z - targets: 2024-02-01T01:07:51.168116055Z - 172.21.249.7 2024-02-01T01:07:51.168118982Z 2024-02-01T01:07:51.171361465Z New config written, updating prometheus 2024-02-01T01:07:56.208294673Z Prometheus config update successful 2024-02-01T01:07:56.221094612Z 2024-02-01T01:08:16.213028688Z Change detected, updating prometheus config 2024-02-01T01:08:16.213049242Z {'_id': {'_data': '8265BAEF00000000052B022C0100296E5A10042C93C5C69BAC4D03B9D4678B057070DB46645F6964006465BAEEE7EE5B0F65E57292AE0004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1706749696, 5), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65baeee7ee5b0f65e57292ae')}} 2024-02-01T01:08:16.215649132Z Jobs detected : 1 2024-02-01T01:08:16.226226112Z Writing new config file to /etc/prometheus/prometheus.yml 2024-02-01T01:08:16.229555664Z alerting: 2024-02-01T01:08:16.229567664Z alertmanagers: 2024-02-01T01:08:16.229571361Z - static_configs: 2024-02-01T01:08:16.229574518Z - targets: null 2024-02-01T01:08:16.229577481Z global: 2024-02-01T01:08:16.229580412Z evaluation_interval: 15s 2024-02-01T01:08:16.229583332Z scrape_interval: 15s 2024-02-01T01:08:16.229586466Z rule_files: null 2024-02-01T01:08:16.229589434Z scrape_configs: 2024-02-01T01:08:16.229596211Z - job_name: mon_exporter 2024-02-01T01:08:16.229599309Z static_configs: 2024-02-01T01:08:16.229602435Z - targets: 2024-02-01T01:08:16.229605337Z - mon:8000 2024-02-01T01:08:16.229608335Z 2024-02-01T01:08:16.231194738Z New config written, updating prometheus 2024-02-01T01:08:21.251278956Z Prometheus config update successful 2024-02-01T01:08:21.252993807Z