2024-01-15T21:29:33.270197902Z level=info ts=2024-01-15T21:29:33.254Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d
2024-01-15T21:29:33.271527400Z level=info ts=2024-01-15T21:29:33.255Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)"
2024-01-15T21:29:33.271540726Z level=info ts=2024-01-15T21:29:33.255Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)"
2024-01-15T21:29:33.271546754Z level=info ts=2024-01-15T21:29:33.255Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))"
2024-01-15T21:29:33.271551912Z level=info ts=2024-01-15T21:29:33.255Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)"
2024-01-15T21:29:33.271572112Z level=info ts=2024-01-15T21:29:33.255Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)"
2024-01-15T21:29:33.271576385Z level=info ts=2024-01-15T21:29:33.257Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-01-15T21:29:33.271579642Z level=info ts=2024-01-15T21:29:33.258Z caller=main.go:824 msg="Starting TSDB ..."
2024-01-15T21:29:33.271582754Z level=info ts=2024-01-15T21:29:33.260Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false
2024-01-15T21:29:33.271585943Z level=info ts=2024-01-15T21:29:33.264Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-01-15T21:29:33.271589241Z level=info ts=2024-01-15T21:29:33.264Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.297µs
2024-01-15T21:29:33.271593033Z level=info ts=2024-01-15T21:29:33.264Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while"
2024-01-15T21:29:33.271596019Z level=info ts=2024-01-15T21:29:33.264Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-01-15T21:29:33.271599154Z level=info ts=2024-01-15T21:29:33.264Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.236µs wal_replay_duration=210.043µs total_replay_duration=261.146µs
2024-01-15T21:29:33.271603854Z level=info ts=2024-01-15T21:29:33.266Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC
2024-01-15T21:29:33.271606835Z level=info ts=2024-01-15T21:29:33.266Z caller=main.go:854 msg="TSDB started"
2024-01-15T21:29:33.271612194Z level=info ts=2024-01-15T21:29:33.266Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-15T21:29:33.271615574Z level=info ts=2024-01-15T21:29:33.267Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=684.483µs remote_storage=5.78µs web_handler=744ns query_engine=3.325µs scrape=270.96µs scrape_sd=28.229µs notify=27.432µs notify_sd=15.859µs rules=3.97µs
2024-01-15T21:29:33.271619372Z level=info ts=2024-01-15T21:29:33.267Z caller=main.go:796 msg="Server is ready to receive web requests."
2024-01-15T21:33:50.420973779Z level=info ts=2024-01-15T21:33:50.420Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-15T21:33:50.422012572Z level=info ts=2024-01-15T21:33:50.421Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.60697ms remote_storage=3.304µs web_handler=7.413µs query_engine=6.571µs scrape=293.862µs scrape_sd=37.934µs notify=22.816µs notify_sd=11.7µs rules=8.1µs
2024-01-15T21:34:00.471397525Z level=info ts=2024-01-15T21:34:00.471Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-15T21:34:00.473134503Z level=info ts=2024-01-15T21:34:00.472Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.8702ms remote_storage=4.877µs web_handler=12.998µs query_engine=11.043µs scrape=194.715µs scrape_sd=63.934µs notify=62.438µs notify_sd=37.073µs rules=24.642µs
2024-01-15T21:34:10.537035423Z level=info ts=2024-01-15T21:34:10.536Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-15T21:34:10.539459947Z level=info ts=2024-01-15T21:34:10.538Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.385474ms remote_storage=4.546µs web_handler=1.369µs query_engine=3.521µs scrape=358.594µs scrape_sd=91.43µs notify=96.28µs notify_sd=115.612µs rules=2.991µs
2024-01-16T00:29:54.180028211Z level=info ts=2024-01-16T00:29:54.177Z caller=compact.go:518 component=tsdb msg="write block" mint=1705354179124 maxt=1705356000000 ulid=01HM7TSE2HEM1XVHX21Y2RD05S duration=48.646744ms
2024-01-16T00:29:54.182234381Z level=info ts=2024-01-16T00:29:54.182Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=3.005258ms
2024-01-16T01:00:24.184345545Z level=info ts=2024-01-16T01:00:24.183Z caller=compact.go:518 component=tsdb msg="write block" mint=1705356009124 maxt=1705363200000 ulid=01HM7WH961X2WM3MMVH88QHH0V duration=54.407628ms
2024-01-16T01:00:24.187621171Z level=info ts=2024-01-16T01:00:24.187Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=2.406474ms
2024-01-16T01:05:38.146659017Z level=info ts=2024-01-16T01:05:38.146Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-16T01:05:38.147840250Z level=info ts=2024-01-16T01:05:38.147Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.342463ms remote_storage=3.251µs web_handler=5.416µs query_engine=5.422µs scrape=150.186µs scrape_sd=77.494µs notify=38.374µs notify_sd=17.052µs rules=9.357µs
2024-01-16T01:06:04.679912402Z level=info ts=2024-01-16T01:06:04.676Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-16T01:06:04.679969575Z level=info ts=2024-01-16T01:06:04.677Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.056123ms remote_storage=23.633µs web_handler=635ns query_engine=1.589µs scrape=378.392µs scrape_sd=59.666µs notify=18.77µs notify_sd=34.24µs rules=1.966µs
2024-01-15T21:33:49.099762964Z Created MongoClient to connect to MongoDB!
2024-01-15T21:33:49.099844633Z Refreshing prometheus config file for first time
2024-01-15T21:33:49.392895348Z Jobs detected : 0
2024-01-15T21:33:49.398437739Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-15T21:33:49.399662848Z alerting:
2024-01-15T21:33:49.399674311Z   alertmanagers:
2024-01-15T21:33:49.399678154Z   - static_configs:
2024-01-15T21:33:49.399681697Z     - targets: null
2024-01-15T21:33:49.399685128Z global:
2024-01-15T21:33:49.399688634Z   evaluation_interval: 15s
2024-01-15T21:33:49.399692170Z   scrape_interval: 15s
2024-01-15T21:33:49.399695865Z rule_files: null
2024-01-15T21:33:49.399699283Z scrape_configs:
2024-01-15T21:33:49.399702688Z - job_name: mon_exporter
2024-01-15T21:33:49.399706102Z   static_configs:
2024-01-15T21:33:49.399709472Z   - targets:
2024-01-15T21:33:49.399712797Z     - mon:8000
2024-01-15T21:33:49.399716130Z 
2024-01-15T21:33:49.401922032Z New config written, updating prometheus
2024-01-15T21:33:55.445841225Z Prometheus config update successful
2024-01-15T21:34:00.448504773Z Refreshing prometheus config file for first time
2024-01-15T21:34:00.452943726Z Jobs detected : 1
2024-01-15T21:34:00.463785901Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-15T21:34:00.465801537Z alerting:
2024-01-15T21:34:00.465830562Z   alertmanagers:
2024-01-15T21:34:00.465839865Z   - static_configs:
2024-01-15T21:34:00.465848150Z     - targets: null
2024-01-15T21:34:00.465856290Z global:
2024-01-15T21:34:00.465864174Z   evaluation_interval: 15s
2024-01-15T21:34:00.465872701Z   scrape_interval: 15s
2024-01-15T21:34:00.465880946Z rule_files: null
2024-01-15T21:34:00.465888804Z scrape_configs:
2024-01-15T21:34:00.465896651Z - job_name: mon_exporter
2024-01-15T21:34:00.465904580Z   static_configs:
2024-01-15T21:34:00.465912648Z   - targets:
2024-01-15T21:34:00.465920511Z     - mon:8000
2024-01-15T21:34:00.465928376Z 
2024-01-15T21:34:00.467721703Z New config written, updating prometheus
2024-01-15T21:34:05.504463779Z Prometheus config update successful
2024-01-15T21:34:10.510587747Z Refreshing prometheus config file for first time
2024-01-15T21:34:10.513826042Z Jobs detected : 1
2024-01-15T21:34:10.525747264Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-15T21:34:10.530489167Z alerting:
2024-01-15T21:34:10.530524405Z   alertmanagers:
2024-01-15T21:34:10.530577647Z   - static_configs:
2024-01-15T21:34:10.530588049Z     - targets: null
2024-01-15T21:34:10.530597151Z global:
2024-01-15T21:34:10.530605412Z   evaluation_interval: 15s
2024-01-15T21:34:10.530613356Z   scrape_interval: 15s
2024-01-15T21:34:10.530621272Z rule_files: null
2024-01-15T21:34:10.530629108Z scrape_configs:
2024-01-15T21:34:10.530637068Z - job_name: mon_exporter
2024-01-15T21:34:10.530644947Z   static_configs:
2024-01-15T21:34:10.530652837Z   - targets:
2024-01-15T21:34:10.530660966Z     - mon:8000
2024-01-15T21:34:10.530669126Z 
2024-01-15T21:34:10.532481118Z New config written, updating prometheus
2024-01-15T21:34:15.554442447Z Prometheus config update successful
2024-01-15T21:34:20.562016457Z Listening to changes in prometheus jobs collection
2024-01-16T01:05:38.123097087Z Change detected, updating prometheus config
2024-01-16T01:05:38.123300578Z {'_id': {'_data': '8265A5D662000000032B022C0100296E5A10045EB90893CA6F479DB04DA0FE2AD1331446645F6964006465A5D6620E770ABAB6CC223C0004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1705367138, 3), 'fullDocument': {'_id': ObjectId('65a5d6620e770abab6cc223c'), 'job_name': '1de488b8266f40de939341c684ef769b', 'metrics_path': '/snmp', 'nsr_id': '0795cbcd-f423-4ba7-a2ee-6d79a759b161', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0096778482-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.249.81']}], 'vnfr_id': '1de488b8266f40de939341c684ef769b'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65a5d6620e770abab6cc223c')}}
2024-01-16T01:05:38.126864220Z Jobs detected : 2
2024-01-16T01:05:38.134541286Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-16T01:05:38.137901981Z alerting:
2024-01-16T01:05:38.137916264Z   alertmanagers:
2024-01-16T01:05:38.137920023Z   - static_configs:
2024-01-16T01:05:38.137923735Z     - targets: null
2024-01-16T01:05:38.137926977Z global:
2024-01-16T01:05:38.137930168Z   evaluation_interval: 15s
2024-01-16T01:05:38.137933374Z   scrape_interval: 15s
2024-01-16T01:05:38.137936879Z rule_files: null
2024-01-16T01:05:38.137940033Z scrape_configs:
2024-01-16T01:05:38.137943294Z - job_name: mon_exporter
2024-01-16T01:05:38.137946430Z   static_configs:
2024-01-16T01:05:38.137949713Z   - targets:
2024-01-16T01:05:38.137953477Z     - mon:8000
2024-01-16T01:05:38.137957014Z - job_name: 1de488b8266f40de939341c684ef769b
2024-01-16T01:05:38.137960188Z   metrics_path: /snmp
2024-01-16T01:05:38.137963359Z   params:
2024-01-16T01:05:38.137981774Z     module:
2024-01-16T01:05:38.137984699Z     - osm-snmp
2024-01-16T01:05:38.137987447Z   relabel_configs:
2024-01-16T01:05:38.137990172Z   - source_labels:
2024-01-16T01:05:38.137992893Z     - __address__
2024-01-16T01:05:38.137995645Z     target_label: __param_target
2024-01-16T01:05:38.137998426Z   - source_labels:
2024-01-16T01:05:38.138001215Z     - __param_target
2024-01-16T01:05:38.138003947Z     target_label: instance
2024-01-16T01:05:38.138006707Z   - replacement: eechart-0096778482-snmpexporter:80
2024-01-16T01:05:38.138009840Z     target_label: __address__
2024-01-16T01:05:38.138012771Z   static_configs:
2024-01-16T01:05:38.138015487Z   - targets:
2024-01-16T01:05:38.138018236Z     - 172.21.249.81
2024-01-16T01:05:38.138021006Z 
2024-01-16T01:05:38.140942388Z New config written, updating prometheus
2024-01-16T01:05:43.181642066Z Prometheus config update successful
2024-01-16T01:05:43.185420375Z 
2024-01-16T01:06:04.653394422Z Change detected, updating prometheus config
2024-01-16T01:06:04.653435089Z {'_id': {'_data': '8265A5D67C000000082B022C0100296E5A10045EB90893CA6F479DB04DA0FE2AD1331446645F6964006465A5D6620E770ABAB6CC223C0004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1705367164, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65a5d6620e770abab6cc223c')}}
2024-01-16T01:06:04.656055803Z Jobs detected : 1
2024-01-16T01:06:04.665002422Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-16T01:06:04.667552618Z alerting:
2024-01-16T01:06:04.667571420Z   alertmanagers:
2024-01-16T01:06:04.667599212Z   - static_configs:
2024-01-16T01:06:04.667606290Z     - targets: null
2024-01-16T01:06:04.667612038Z global:
2024-01-16T01:06:04.667617658Z   evaluation_interval: 15s
2024-01-16T01:06:04.667623450Z   scrape_interval: 15s
2024-01-16T01:06:04.667629246Z rule_files: null
2024-01-16T01:06:04.667634824Z scrape_configs:
2024-01-16T01:06:04.667640497Z - job_name: mon_exporter
2024-01-16T01:06:04.667646270Z   static_configs:
2024-01-16T01:06:04.667651985Z   - targets:
2024-01-16T01:06:04.667657720Z     - mon:8000
2024-01-16T01:06:04.667681507Z 
2024-01-16T01:06:04.670059729Z New config written, updating prometheus
2024-01-16T01:06:09.703298606Z Prometheus config update successful
2024-01-16T01:06:09.705713414Z