2024-01-27T21:30:17.776354976Z level=info ts=2024-01-27T21:30:17.775Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d
2024-01-27T21:30:17.776769838Z level=info ts=2024-01-27T21:30:17.776Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)"
2024-01-27T21:30:17.776874378Z level=info ts=2024-01-27T21:30:17.776Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)"
2024-01-27T21:30:17.777623521Z level=info ts=2024-01-27T21:30:17.777Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))"
2024-01-27T21:30:17.777700175Z level=info ts=2024-01-27T21:30:17.777Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)"
2024-01-27T21:30:17.777878188Z level=info ts=2024-01-27T21:30:17.777Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)"
2024-01-27T21:30:17.782580278Z level=info ts=2024-01-27T21:30:17.782Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-01-27T21:30:17.784963865Z level=info ts=2024-01-27T21:30:17.784Z caller=main.go:824 msg="Starting TSDB ..."
2024-01-27T21:30:17.789876861Z level=info ts=2024-01-27T21:30:17.789Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false
2024-01-27T21:30:17.791098374Z level=info ts=2024-01-27T21:30:17.791Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-01-27T21:30:17.791178203Z level=info ts=2024-01-27T21:30:17.791Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=7.185µs
2024-01-27T21:30:17.791184973Z level=info ts=2024-01-27T21:30:17.791Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while"
2024-01-27T21:30:17.791473691Z level=info ts=2024-01-27T21:30:17.791Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-01-27T21:30:17.791481169Z level=info ts=2024-01-27T21:30:17.791Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.068µs wal_replay_duration=238.889µs total_replay_duration=323.312µs
2024-01-27T21:30:17.792933639Z level=info ts=2024-01-27T21:30:17.792Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC
2024-01-27T21:30:17.792941606Z level=info ts=2024-01-27T21:30:17.792Z caller=main.go:854 msg="TSDB started"
2024-01-27T21:30:17.793004076Z level=info ts=2024-01-27T21:30:17.792Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-27T21:30:17.793660036Z level=info ts=2024-01-27T21:30:17.793Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=636.844µs remote_storage=2.65µs web_handler=461ns query_engine=6.585µs scrape=255.487µs scrape_sd=25.283µs notify=25.929µs notify_sd=9.217µs rules=3.559µs
2024-01-27T21:30:17.793673329Z level=info ts=2024-01-27T21:30:17.793Z caller=main.go:796 msg="Server is ready to receive web requests."
2024-01-27T21:36:47.242084420Z level=info ts=2024-01-27T21:36:47.241Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-27T21:36:47.243286177Z level=info ts=2024-01-27T21:36:47.243Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.271579ms remote_storage=3.24µs web_handler=4.657µs query_engine=5.901µs scrape=302.161µs scrape_sd=51.94µs notify=52.179µs notify_sd=15.522µs rules=8.67µs
2024-01-27T21:36:57.288261820Z level=info ts=2024-01-27T21:36:57.288Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-27T21:36:57.289149664Z level=info ts=2024-01-27T21:36:57.288Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=944.251µs remote_storage=3.064µs web_handler=1.048µs query_engine=1.586µs scrape=142.215µs scrape_sd=81.62µs notify=19.706µs notify_sd=39.701µs rules=1.829µs
2024-01-27T21:37:07.341157792Z level=info ts=2024-01-27T21:37:07.340Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-27T21:37:07.343254236Z level=info ts=2024-01-27T21:37:07.343Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.217684ms remote_storage=3.673µs web_handler=833ns query_engine=2.021µs scrape=234.518µs scrape_sd=64.95µs notify=22.527µs notify_sd=31.664µs rules=2.224µs
2024-01-28T00:30:39.155183874Z level=info ts=2024-01-28T00:30:39.154Z caller=compact.go:518 component=tsdb msg="write block" mint=1706391024125 maxt=1706392800000 ulid=01HN6QKE0S42MCSEGF7E7ERWAC duration=24.89211ms
2024-01-28T00:30:39.157365661Z level=info ts=2024-01-28T00:30:39.157Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.845098ms
2024-01-28T01:00:24.178462884Z level=info ts=2024-01-28T01:00:24.178Z caller=compact.go:518 component=tsdb msg="write block" mint=1706392809125 maxt=1706400000000 ulid=01HN6S9X6670WWPS23CVWNPWJP duration=43.913023ms
2024-01-28T01:00:24.181579378Z level=info ts=2024-01-28T01:00:24.181Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.421806ms
2024-01-28T01:08:21.706047054Z level=info ts=2024-01-28T01:08:21.698Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-28T01:08:21.706072644Z level=info ts=2024-01-28T01:08:21.703Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.670017ms remote_storage=2.907µs web_handler=3.653µs query_engine=2.008µs scrape=153.533µs scrape_sd=71.442µs notify=45.288µs notify_sd=15.122µs rules=7.911µs
2024-01-28T01:08:47.329842304Z level=info ts=2024-01-28T01:08:47.329Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-28T01:08:47.329873623Z level=info ts=2024-01-28T01:08:47.329Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=709.538µs remote_storage=2.112µs web_handler=786ns query_engine=1.613µs scrape=193.463µs scrape_sd=26.166µs notify=14.796µs notify_sd=9.958µs rules=1.663µs
2024-01-27T21:36:47.217835821Z Created MongoClient to connect to MongoDB!
2024-01-27T21:36:47.217887545Z Refreshing prometheus config file for first time
2024-01-27T21:36:47.231181037Z Jobs detected : 1
2024-01-27T21:36:47.234907987Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-27T21:36:47.236780801Z alerting:
2024-01-27T21:36:47.236798161Z   alertmanagers:
2024-01-27T21:36:47.236803403Z   - static_configs:
2024-01-27T21:36:47.236808021Z     - targets: null
2024-01-27T21:36:47.236812231Z global:
2024-01-27T21:36:47.236816556Z   evaluation_interval: 15s
2024-01-27T21:36:47.236820839Z   scrape_interval: 15s
2024-01-27T21:36:47.236825273Z rule_files: null
2024-01-27T21:36:47.236829567Z scrape_configs:
2024-01-27T21:36:47.236833872Z - job_name: mon_exporter
2024-01-27T21:36:47.236838179Z   static_configs:
2024-01-27T21:36:47.236842699Z   - targets:
2024-01-27T21:36:47.236846902Z     - mon:8000
2024-01-27T21:36:47.236851331Z 
2024-01-27T21:36:47.237915978Z New config written, updating prometheus
2024-01-27T21:36:52.259488506Z Prometheus config update successful
2024-01-27T21:36:57.265855669Z Refreshing prometheus config file for first time
2024-01-27T21:36:57.267732413Z Jobs detected : 1
2024-01-27T21:36:57.278616797Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-27T21:36:57.281671776Z alerting:
2024-01-27T21:36:57.281694961Z   alertmanagers:
2024-01-27T21:36:57.281701172Z   - static_configs:
2024-01-27T21:36:57.281706202Z     - targets: null
2024-01-27T21:36:57.281711292Z global:
2024-01-27T21:36:57.281716140Z   evaluation_interval: 15s
2024-01-27T21:36:57.281721778Z   scrape_interval: 15s
2024-01-27T21:36:57.281726596Z rule_files: null
2024-01-27T21:36:57.281731369Z scrape_configs:
2024-01-27T21:36:57.281736232Z - job_name: mon_exporter
2024-01-27T21:36:57.281741204Z   static_configs:
2024-01-27T21:36:57.281746060Z   - targets:
2024-01-27T21:36:57.281750975Z     - mon:8000
2024-01-27T21:36:57.281755772Z 
2024-01-27T21:36:57.283727885Z New config written, updating prometheus
2024-01-27T21:37:02.313366051Z Prometheus config update successful
2024-01-27T21:37:07.319125653Z Refreshing prometheus config file for first time
2024-01-27T21:37:07.321525852Z Jobs detected : 1
2024-01-27T21:37:07.332809936Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-27T21:37:07.334721066Z alerting:
2024-01-27T21:37:07.334757490Z   alertmanagers:
2024-01-27T21:37:07.334780817Z   - static_configs:
2024-01-27T21:37:07.334785260Z     - targets: null
2024-01-27T21:37:07.334788876Z global:
2024-01-27T21:37:07.334792491Z   evaluation_interval: 15s
2024-01-27T21:37:07.334796069Z   scrape_interval: 15s
2024-01-27T21:37:07.334799684Z rule_files: null
2024-01-27T21:37:07.334803490Z scrape_configs:
2024-01-27T21:37:07.334807078Z - job_name: mon_exporter
2024-01-27T21:37:07.334810659Z   static_configs:
2024-01-27T21:37:07.334829100Z   - targets:
2024-01-27T21:37:07.334833126Z     - mon:8000
2024-01-27T21:37:07.334836749Z 
2024-01-27T21:37:07.336470071Z New config written, updating prometheus
2024-01-27T21:37:12.365677297Z Prometheus config update successful
2024-01-27T21:37:17.375548423Z Listening to changes in prometheus jobs collection
2024-01-28T01:08:21.659652120Z Change detected, updating prometheus config
2024-01-28T01:08:21.659687751Z {'_id': {'_data': '8265B5A905000000042B022C0100296E5A10045F06121D07624D799C30B0F068DC1A0046645F6964006465B5A9058102C69058AFD0340004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1706404101, 4), 'fullDocument': {'_id': ObjectId('65b5a9058102c69058afd034'), 'job_name': '039b3cbb2ed54aa68a9638b145d37e66', 'metrics_path': '/snmp', 'nsr_id': 'b4f989c3-3680-43e5-8d2a-f807dfd934d5', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0083854134-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.248.247']}], 'vnfr_id': '039b3cbb2ed54aa68a9638b145d37e66'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65b5a9058102c69058afd034')}}
2024-01-28T01:08:21.668473827Z Jobs detected : 2
2024-01-28T01:08:21.674397323Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-28T01:08:21.684359448Z alerting:
2024-01-28T01:08:21.684403042Z   alertmanagers:
2024-01-28T01:08:21.684409556Z   - static_configs:
2024-01-28T01:08:21.684414566Z     - targets: null
2024-01-28T01:08:21.684419693Z global:
2024-01-28T01:08:21.684424340Z   evaluation_interval: 15s
2024-01-28T01:08:21.684434687Z   scrape_interval: 15s
2024-01-28T01:08:21.684439416Z rule_files: null
2024-01-28T01:08:21.684443968Z scrape_configs:
2024-01-28T01:08:21.684448539Z - job_name: mon_exporter
2024-01-28T01:08:21.684453140Z   static_configs:
2024-01-28T01:08:21.684457723Z   - targets:
2024-01-28T01:08:21.684480465Z     - mon:8000
2024-01-28T01:08:21.684485105Z - job_name: 039b3cbb2ed54aa68a9638b145d37e66
2024-01-28T01:08:21.684489961Z   metrics_path: /snmp
2024-01-28T01:08:21.684494513Z   params:
2024-01-28T01:08:21.684513212Z     module:
2024-01-28T01:08:21.684517461Z     - osm-snmp
2024-01-28T01:08:21.684521536Z   relabel_configs:
2024-01-28T01:08:21.684525468Z   - source_labels:
2024-01-28T01:08:21.684529527Z     - __address__
2024-01-28T01:08:21.684533487Z     target_label: __param_target
2024-01-28T01:08:21.684555109Z   - source_labels:
2024-01-28T01:08:21.684559724Z     - __param_target
2024-01-28T01:08:21.684563812Z     target_label: instance
2024-01-28T01:08:21.684567849Z   - replacement: eechart-0083854134-snmpexporter:80
2024-01-28T01:08:21.684572047Z     target_label: __address__
2024-01-28T01:08:21.684576016Z   static_configs:
2024-01-28T01:08:21.684580199Z   - targets:
2024-01-28T01:08:21.684584320Z     - 172.21.248.247
2024-01-28T01:08:21.684588412Z 
2024-01-28T01:08:21.692510521Z New config written, updating prometheus
2024-01-28T01:08:26.738902106Z Prometheus config update successful
2024-01-28T01:08:26.743031004Z 
2024-01-28T01:08:47.312727332Z Change detected, updating prometheus config
2024-01-28T01:08:47.312939497Z {'_id': {'_data': '8265B5A91F000000082B022C0100296E5A10045F06121D07624D799C30B0F068DC1A0046645F6964006465B5A9058102C69058AFD0340004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1706404127, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65b5a9058102c69058afd034')}}
2024-01-28T01:08:47.314756386Z Jobs detected : 1
2024-01-28T01:08:47.319341533Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-28T01:08:47.321339149Z alerting:
2024-01-28T01:08:47.321352222Z   alertmanagers:
2024-01-28T01:08:47.321356782Z   - static_configs:
2024-01-28T01:08:47.321360500Z     - targets: null
2024-01-28T01:08:47.321364210Z global:
2024-01-28T01:08:47.321367894Z   evaluation_interval: 15s
2024-01-28T01:08:47.321371477Z   scrape_interval: 15s
2024-01-28T01:08:47.321375056Z rule_files: null
2024-01-28T01:08:47.321378551Z scrape_configs:
2024-01-28T01:08:47.321382090Z - job_name: mon_exporter
2024-01-28T01:08:47.321385695Z   static_configs:
2024-01-28T01:08:47.321389203Z   - targets:
2024-01-28T01:08:47.321392669Z     - mon:8000
2024-01-28T01:08:47.321396244Z 
2024-01-28T01:08:47.323769387Z New config written, updating prometheus
2024-01-28T01:08:52.360618179Z Prometheus config update successful
2024-01-28T01:08:52.363462316Z