2023-12-27T21:35:21.418948201Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d
2023-12-27T21:35:21.418996411Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)"
2023-12-27T21:35:21.419001493Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)"
2023-12-27T21:35:21.419004951Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))"
2023-12-27T21:35:21.419008089Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)"
2023-12-27T21:35:21.419011130Z level=info ts=2023-12-27T21:35:21.415Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)"
2023-12-27T21:35:21.421238408Z level=info ts=2023-12-27T21:35:21.420Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090
2023-12-27T21:35:21.423049958Z level=info ts=2023-12-27T21:35:21.422Z caller=main.go:824 msg="Starting TSDB ..."
2023-12-27T21:35:21.427133231Z level=info ts=2023-12-27T21:35:21.424Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false
2023-12-27T21:35:21.441679264Z level=info ts=2023-12-27T21:35:21.429Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2023-12-27T21:35:21.441704699Z level=info ts=2023-12-27T21:35:21.429Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=11.2µs
2023-12-27T21:35:21.441709773Z level=info ts=2023-12-27T21:35:21.429Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while"
2023-12-27T21:35:21.441713056Z level=info ts=2023-12-27T21:35:21.429Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2023-12-27T21:35:21.441716423Z level=info ts=2023-12-27T21:35:21.429Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.298µs wal_replay_duration=317.411µs total_replay_duration=376.705µs
2023-12-27T21:35:21.441720316Z level=info ts=2023-12-27T21:35:21.431Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC
2023-12-27T21:35:21.441723266Z level=info ts=2023-12-27T21:35:21.431Z caller=main.go:854 msg="TSDB started"
2023-12-27T21:35:21.441726160Z level=info ts=2023-12-27T21:35:21.431Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-27T21:35:21.451148495Z level=info ts=2023-12-27T21:35:21.450Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.11788ms remote_storage=11.095µs web_handler=834ns query_engine=5.497µs scrape=18.447562ms scrape_sd=67.277µs notify=61.962µs notify_sd=29.625µs rules=5.323µs
2023-12-27T21:35:21.452189542Z level=info ts=2023-12-27T21:35:21.450Z caller=main.go:796 msg="Server is ready to receive web requests."
2023-12-27T21:40:31.741138243Z level=info ts=2023-12-27T21:40:31.740Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-27T21:40:31.745492317Z level=info ts=2023-12-27T21:40:31.745Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.648821ms remote_storage=11.23µs web_handler=14.974µs query_engine=11.116µs scrape=254.333µs scrape_sd=137.818µs notify=68.661µs notify_sd=37.782µs rules=20.617µs
2023-12-27T21:40:41.802218422Z level=info ts=2023-12-27T21:40:41.801Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-27T21:40:41.803716960Z level=info ts=2023-12-27T21:40:41.803Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.439518ms remote_storage=3.841µs web_handler=1.21µs query_engine=2.787µs scrape=238.74µs scrape_sd=56.35µs notify=24.829µs notify_sd=90.889µs rules=2.644µs
2023-12-28T00:35:54.176454004Z level=info ts=2023-12-28T00:35:54.175Z caller=compact.go:518 component=tsdb msg="write block" mint=1703712939123 maxt=1703714400000 ulid=01HJPXJRMK6HXFTW0PVH0MV723 duration=43.593502ms
2023-12-28T00:35:54.183354593Z level=info ts=2023-12-28T00:35:54.183Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=5.441377ms
2023-12-28T01:00:24.197028138Z level=info ts=2023-12-28T01:00:24.196Z caller=compact.go:518 component=tsdb msg="write block" mint=1703714409123 maxt=1703721600000 ulid=01HJPYZM67RGJNV34Z26S47QNZ duration=60.769419ms
2023-12-28T01:00:24.202992674Z level=info ts=2023-12-28T01:00:24.202Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=4.904562ms
2023-12-28T01:22:18.012833225Z level=info ts=2023-12-28T01:22:18.012Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-28T01:22:18.023241276Z level=info ts=2023-12-28T01:22:18.022Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=10.55964ms remote_storage=9.681µs web_handler=4.438µs query_engine=12.893µs scrape=2.670796ms scrape_sd=134.848µs notify=82.619µs notify_sd=17.742µs rules=27.083µs
2023-12-28T01:22:42.631446694Z level=info ts=2023-12-28T01:22:42.631Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-28T01:22:42.632369707Z level=info ts=2023-12-28T01:22:42.632Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=956.211µs remote_storage=3.472µs web_handler=870ns query_engine=2.122µs scrape=554.791µs scrape_sd=29.263µs notify=15.198µs notify_sd=11.831µs rules=1.467µs
2023-12-27T21:39:53.881740221Z Created MongoClient to connect to MongoDB!
2023-12-27T21:39:53.881787321Z Refreshing prometheus config file for first time
2023-12-27T21:40:23.934369106Z Error in configuration attempt! Number of tries: 1/3
2023-12-27T21:40:23.934440057Z No replica set members match selector "<function readable_server_selector at 0x7fb33bc334c0>", Timeout: 30s, Topology Description: <TopologyDescription id: 658c99a9fea3a11f2bffdc55, topology_type: ReplicaSetNoPrimary, servers: [<ServerDescription ('mongodb-k8s', 27017) server_type: RSGhost, rtt: 0.0030899139521134208>]>
2023-12-27T21:40:28.938931118Z Refreshing prometheus config file for first time
2023-12-27T21:40:31.710206713Z Jobs detected : 0
2023-12-27T21:40:31.724716600Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-27T21:40:31.727439268Z alerting:
2023-12-27T21:40:31.727454430Z   alertmanagers:
2023-12-27T21:40:31.727458339Z   - static_configs:
2023-12-27T21:40:31.727461448Z     - targets: null
2023-12-27T21:40:31.727464727Z global:
2023-12-27T21:40:31.727467615Z   evaluation_interval: 15s
2023-12-27T21:40:31.727470579Z   scrape_interval: 15s
2023-12-27T21:40:31.727473559Z rule_files: null
2023-12-27T21:40:31.727476377Z scrape_configs:
2023-12-27T21:40:31.727479203Z - job_name: mon_exporter
2023-12-27T21:40:31.727482126Z   static_configs:
2023-12-27T21:40:31.727485056Z   - targets:
2023-12-27T21:40:31.727487930Z     - mon:8000
2023-12-27T21:40:31.727734370Z 
2023-12-27T21:40:31.731282906Z New config written, updating prometheus
2023-12-27T21:40:36.773707260Z Prometheus config update successful
2023-12-27T21:40:41.778964137Z Refreshing prometheus config file for first time
2023-12-27T21:40:41.783158496Z Jobs detected : 1
2023-12-27T21:40:41.793559993Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-27T21:40:41.796169686Z alerting:
2023-12-27T21:40:41.796192751Z   alertmanagers:
2023-12-27T21:40:41.796202878Z   - static_configs:
2023-12-27T21:40:41.796212228Z     - targets: null
2023-12-27T21:40:41.796221023Z global:
2023-12-27T21:40:41.796230418Z   evaluation_interval: 15s
2023-12-27T21:40:41.796240176Z   scrape_interval: 15s
2023-12-27T21:40:41.796249573Z rule_files: null
2023-12-27T21:40:41.796258350Z scrape_configs:
2023-12-27T21:40:41.796267257Z - job_name: mon_exporter
2023-12-27T21:40:41.796276164Z   static_configs:
2023-12-27T21:40:41.796285176Z   - targets:
2023-12-27T21:40:41.796293939Z     - mon:8000
2023-12-27T21:40:41.796303005Z 
2023-12-27T21:40:41.798408247Z New config written, updating prometheus
2023-12-27T21:40:46.836269502Z Prometheus config update successful
2023-12-27T21:40:51.848205890Z Listening to changes in prometheus jobs collection
2023-12-28T01:22:17.989210757Z Change detected, updating prometheus config
2023-12-28T01:22:17.989297220Z {'_id': {'_data': '82658CCDC9000000042B022C0100296E5A1004E1081256853744ADAAB395FC0831A75F46645F69640064658CCDC96589D300635BE1A20004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1703726537, 4), 'fullDocument': {'_id': ObjectId('658ccdc96589d300635be1a2'), 'job_name': '2ed4f01d97324dbf8ca4c5f7e3cf3447', 'metrics_path': '/snmp', 'nsr_id': 'af239549-4ae2-4059-929d-21965cd60a34', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0093784245-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.248.112']}], 'vnfr_id': '2ed4f01d97324dbf8ca4c5f7e3cf3447'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('658ccdc96589d300635be1a2')}}
2023-12-28T01:22:17.992279488Z Jobs detected : 2
2023-12-28T01:22:17.998154618Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-28T01:22:18.001441267Z alerting:
2023-12-28T01:22:18.001460143Z   alertmanagers:
2023-12-28T01:22:18.001465936Z   - static_configs:
2023-12-28T01:22:18.001470597Z     - targets: null
2023-12-28T01:22:18.001475129Z global:
2023-12-28T01:22:18.001480021Z   evaluation_interval: 15s
2023-12-28T01:22:18.001484575Z   scrape_interval: 15s
2023-12-28T01:22:18.001489617Z rule_files: null
2023-12-28T01:22:18.001494157Z scrape_configs:
2023-12-28T01:22:18.001499156Z - job_name: mon_exporter
2023-12-28T01:22:18.001503743Z   static_configs:
2023-12-28T01:22:18.001508256Z   - targets:
2023-12-28T01:22:18.001512696Z     - mon:8000
2023-12-28T01:22:18.001517577Z - job_name: 2ed4f01d97324dbf8ca4c5f7e3cf3447
2023-12-28T01:22:18.001537877Z   metrics_path: /snmp
2023-12-28T01:22:18.001542729Z   params:
2023-12-28T01:22:18.001547041Z     module:
2023-12-28T01:22:18.001551459Z     - osm-snmp
2023-12-28T01:22:18.001556123Z   relabel_configs:
2023-12-28T01:22:18.001560898Z   - source_labels:
2023-12-28T01:22:18.001566110Z     - __address__
2023-12-28T01:22:18.001570992Z     target_label: __param_target
2023-12-28T01:22:18.001576248Z   - source_labels:
2023-12-28T01:22:18.001581092Z     - __param_target
2023-12-28T01:22:18.001585895Z     target_label: instance
2023-12-28T01:22:18.001590817Z   - replacement: eechart-0093784245-snmpexporter:80
2023-12-28T01:22:18.001595676Z     target_label: __address__
2023-12-28T01:22:18.001600905Z   static_configs:
2023-12-28T01:22:18.001626734Z   - targets:
2023-12-28T01:22:18.001631480Z     - 172.21.248.112
2023-12-28T01:22:18.001635898Z 
2023-12-28T01:22:18.003751584Z New config written, updating prometheus
2023-12-28T01:22:23.056050371Z Prometheus config update successful
2023-12-28T01:22:23.060272616Z 
2023-12-28T01:22:42.619372914Z Change detected, updating prometheus config
2023-12-28T01:22:42.619414563Z {'_id': {'_data': '82658CCDE2000000082B022C0100296E5A1004E1081256853744ADAAB395FC0831A75F46645F69640064658CCDC96589D300635BE1A20004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1703726562, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('658ccdc96589d300635be1a2')}}
2023-12-28T01:22:42.620688384Z Jobs detected : 1
2023-12-28T01:22:42.625368444Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-28T01:22:42.626882441Z alerting:
2023-12-28T01:22:42.626897378Z   alertmanagers:
2023-12-28T01:22:42.626901852Z   - static_configs:
2023-12-28T01:22:42.626905768Z     - targets: null
2023-12-28T01:22:42.626909353Z global:
2023-12-28T01:22:42.626912983Z   evaluation_interval: 15s
2023-12-28T01:22:42.626930785Z   scrape_interval: 15s
2023-12-28T01:22:42.626934241Z rule_files: null
2023-12-28T01:22:42.626937700Z scrape_configs:
2023-12-28T01:22:42.626941169Z - job_name: mon_exporter
2023-12-28T01:22:42.626944689Z   static_configs:
2023-12-28T01:22:42.626948154Z   - targets:
2023-12-28T01:22:42.626951642Z     - mon:8000
2023-12-28T01:22:42.626955122Z 
2023-12-28T01:22:42.628304352Z New config written, updating prometheus
2023-12-28T01:22:47.652956351Z Prometheus config update successful
2023-12-28T01:22:47.655944555Z