2023-12-29T21:35:42.182566136Z level=info ts=2023-12-29T21:35:42.182Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d
2023-12-29T21:35:42.182810555Z level=info ts=2023-12-29T21:35:42.182Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)"
2023-12-29T21:35:42.182944361Z level=info ts=2023-12-29T21:35:42.182Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)"
2023-12-29T21:35:42.183163992Z level=info ts=2023-12-29T21:35:42.183Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))"
2023-12-29T21:35:42.183343309Z level=info ts=2023-12-29T21:35:42.183Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)"
2023-12-29T21:35:42.183590813Z level=info ts=2023-12-29T21:35:42.183Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)"
2023-12-29T21:35:42.185931927Z level=info ts=2023-12-29T21:35:42.185Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090
2023-12-29T21:35:42.192304037Z level=info ts=2023-12-29T21:35:42.192Z caller=main.go:824 msg="Starting TSDB ..."
2023-12-29T21:35:42.194555509Z level=info ts=2023-12-29T21:35:42.194Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false
2023-12-29T21:35:42.197959026Z level=info ts=2023-12-29T21:35:42.197Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2023-12-29T21:35:42.197994591Z level=info ts=2023-12-29T21:35:42.197Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=11.534µs
2023-12-29T21:35:42.198003671Z level=info ts=2023-12-29T21:35:42.197Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while"
2023-12-29T21:35:42.205205969Z level=info ts=2023-12-29T21:35:42.204Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2023-12-29T21:35:42.205395871Z level=info ts=2023-12-29T21:35:42.205Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.182µs wal_replay_duration=7.371769ms total_replay_duration=7.429006ms
2023-12-29T21:35:42.208035468Z level=info ts=2023-12-29T21:35:42.207Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC
2023-12-29T21:35:42.209067683Z level=info ts=2023-12-29T21:35:42.208Z caller=main.go:854 msg="TSDB started"
2023-12-29T21:35:42.209397504Z level=info ts=2023-12-29T21:35:42.209Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-29T21:35:42.211172586Z level=info ts=2023-12-29T21:35:42.211Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.792968ms remote_storage=23.007µs web_handler=765ns query_engine=4.362µs scrape=670.855µs scrape_sd=93.565µs notify=67.627µs notify_sd=20.275µs rules=5.588µs
2023-12-29T21:35:42.212373825Z level=info ts=2023-12-29T21:35:42.212Z caller=main.go:796 msg="Server is ready to receive web requests."
2023-12-29T21:40:49.258520370Z level=info ts=2023-12-29T21:40:49.258Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-29T21:40:49.265758854Z level=info ts=2023-12-29T21:40:49.265Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=7.302869ms remote_storage=50.3µs web_handler=13.681µs query_engine=25.483µs scrape=553.571µs scrape_sd=245.628µs notify=134.732µs notify_sd=34.394µs rules=33.251µs
2023-12-29T21:40:59.314967839Z level=info ts=2023-12-29T21:40:59.314Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-29T21:40:59.316228408Z level=info ts=2023-12-29T21:40:59.315Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.21683ms remote_storage=5.322µs web_handler=1.195µs query_engine=2.688µs scrape=251.843µs scrape_sd=57.731µs notify=123.498µs notify_sd=34.552µs rules=2.994µs
2023-12-29T21:41:09.373389057Z level=info ts=2023-12-29T21:41:09.372Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-29T21:41:09.374039354Z level=info ts=2023-12-29T21:41:09.373Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=850.453µs remote_storage=3.526µs web_handler=656ns query_engine=1.807µs scrape=302.404µs scrape_sd=88.102µs notify=35.529µs notify_sd=37.831µs rules=2.299µs
2023-12-30T00:36:09.182334407Z level=info ts=2023-12-30T00:36:09.181Z caller=compact.go:518 component=tsdb msg="write block" mint=1703885754123 maxt=1703887200000 ulid=01HJW2CN9M2P3ET6QGTHX3CW1B duration=40.828589ms
2023-12-30T00:36:09.187062406Z level=info ts=2023-12-30T00:36:09.186Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=4.299929ms
2023-12-30T01:00:24.191160424Z level=info ts=2023-12-30T01:00:24.190Z caller=compact.go:518 component=tsdb msg="write block" mint=1703887209123 maxt=1703894400000 ulid=01HJW3S262XZ91W6HZ0F373D7Q duration=59.498905ms
2023-12-30T01:00:24.198249035Z level=info ts=2023-12-30T01:00:24.197Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=5.147683ms
2023-12-30T01:22:56.079238251Z level=info ts=2023-12-30T01:22:56.078Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-30T01:22:56.083261899Z level=info ts=2023-12-30T01:22:56.083Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.41767ms remote_storage=11.629µs web_handler=9.343µs query_engine=15.314µs scrape=526.675µs scrape_sd=292.777µs notify=239.285µs notify_sd=51.656µs rules=18.815µs
2023-12-30T01:23:20.340217368Z level=info ts=2023-12-30T01:23:20.340Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2023-12-30T01:23:20.340890562Z level=info ts=2023-12-30T01:23:20.340Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=682.25µs remote_storage=2.144µs web_handler=749ns query_engine=1.616µs scrape=197.718µs scrape_sd=26.05µs notify=13.315µs notify_sd=10.202µs rules=1.769µs
2023-12-29T21:40:20.178501918Z Created MongoClient to connect to MongoDB!
2023-12-29T21:40:20.178600317Z Refreshing prometheus config file for first time
2023-12-29T21:40:49.236654144Z Jobs detected : 0
2023-12-29T21:40:49.260045892Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-29T21:40:49.260080227Z alerting:
2023-12-29T21:40:49.260084351Z   alertmanagers:
2023-12-29T21:40:49.260087829Z   - static_configs:
2023-12-29T21:40:49.260091330Z     - targets: null
2023-12-29T21:40:49.260094558Z global:
2023-12-29T21:40:49.260098226Z   evaluation_interval: 15s
2023-12-29T21:40:49.260107195Z   scrape_interval: 15s
2023-12-29T21:40:49.260110469Z rule_files: null
2023-12-29T21:40:49.260113502Z scrape_configs:
2023-12-29T21:40:49.260116622Z - job_name: mon_exporter
2023-12-29T21:40:49.260120124Z   static_configs:
2023-12-29T21:40:49.260123268Z   - targets:
2023-12-29T21:40:49.260126361Z     - mon:8000
2023-12-29T21:40:49.260129596Z 
2023-12-29T21:40:49.260132771Z New config written, updating prometheus
2023-12-29T21:40:54.292003742Z Prometheus config update successful
2023-12-29T21:40:59.294733674Z Refreshing prometheus config file for first time
2023-12-29T21:40:59.297819919Z Jobs detected : 1
2023-12-29T21:40:59.306004120Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-29T21:40:59.309014861Z alerting:
2023-12-29T21:40:59.309039746Z   alertmanagers:
2023-12-29T21:40:59.309045618Z   - static_configs:
2023-12-29T21:40:59.309050213Z     - targets: null
2023-12-29T21:40:59.309054722Z global:
2023-12-29T21:40:59.309059795Z   evaluation_interval: 15s
2023-12-29T21:40:59.309066000Z   scrape_interval: 15s
2023-12-29T21:40:59.309072304Z rule_files: null
2023-12-29T21:40:59.309077987Z scrape_configs:
2023-12-29T21:40:59.309083589Z - job_name: mon_exporter
2023-12-29T21:40:59.309089181Z   static_configs:
2023-12-29T21:40:59.309093996Z   - targets:
2023-12-29T21:40:59.309098445Z     - mon:8000
2023-12-29T21:40:59.309103413Z 
2023-12-29T21:40:59.311065545Z New config written, updating prometheus
2023-12-29T21:41:04.341075761Z Prometheus config update successful
2023-12-29T21:41:09.349080852Z Refreshing prometheus config file for first time
2023-12-29T21:41:09.352574513Z Jobs detected : 1
2023-12-29T21:41:09.363698332Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-29T21:41:09.366431572Z alerting:
2023-12-29T21:41:09.366464637Z   alertmanagers:
2023-12-29T21:41:09.366571862Z   - static_configs:
2023-12-29T21:41:09.366581941Z     - targets: null
2023-12-29T21:41:09.366590029Z global:
2023-12-29T21:41:09.366598935Z   evaluation_interval: 15s
2023-12-29T21:41:09.366607164Z   scrape_interval: 15s
2023-12-29T21:41:09.366615399Z rule_files: null
2023-12-29T21:41:09.366623439Z scrape_configs:
2023-12-29T21:41:09.366631560Z - job_name: mon_exporter
2023-12-29T21:41:09.366639908Z   static_configs:
2023-12-29T21:41:09.366648235Z   - targets:
2023-12-29T21:41:09.366656449Z     - mon:8000
2023-12-29T21:41:09.366664811Z 
2023-12-29T21:41:09.368772820Z New config written, updating prometheus
2023-12-29T21:41:14.398941065Z Prometheus config update successful
2023-12-29T21:41:19.410983077Z Listening to changes in prometheus jobs collection
2023-12-30T01:22:56.044041430Z Change detected, updating prometheus config
2023-12-30T01:22:56.044867018Z {'_id': {'_data': '82658F70EF000000042B022C0100296E5A1004AA4C5DE2B010495E9291FE6700CF3E2A46645F69640064658F70EF48EAB65B9A051C9D0004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1703899375, 4), 'fullDocument': {'_id': ObjectId('658f70ef48eab65b9a051c9d'), 'job_name': '83eb872de6a04be896346ee186c9f635', 'metrics_path': '/snmp', 'nsr_id': 'e9cfd9a7-494d-480d-93ac-6b23020148a9', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0077778702-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.249.236']}], 'vnfr_id': '83eb872de6a04be896346ee186c9f635'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('658f70ef48eab65b9a051c9d')}}
2023-12-30T01:22:56.052305460Z Jobs detected : 2
2023-12-30T01:22:56.062455693Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-30T01:22:56.066831388Z alerting:
2023-12-30T01:22:56.066848417Z   alertmanagers:
2023-12-30T01:22:56.066894189Z   - static_configs:
2023-12-30T01:22:56.066900202Z     - targets: null
2023-12-30T01:22:56.066904537Z global:
2023-12-30T01:22:56.066908811Z   evaluation_interval: 15s
2023-12-30T01:22:56.066913170Z   scrape_interval: 15s
2023-12-30T01:22:56.066917525Z rule_files: null
2023-12-30T01:22:56.066922001Z scrape_configs:
2023-12-30T01:22:56.066926253Z - job_name: mon_exporter
2023-12-30T01:22:56.066930510Z   static_configs:
2023-12-30T01:22:56.066934711Z   - targets:
2023-12-30T01:22:56.066938906Z     - mon:8000
2023-12-30T01:22:56.066943257Z - job_name: 83eb872de6a04be896346ee186c9f635
2023-12-30T01:22:56.066963100Z   metrics_path: /snmp
2023-12-30T01:22:56.066967458Z   params:
2023-12-30T01:22:56.066990124Z     module:
2023-12-30T01:22:56.066994243Z     - osm-snmp
2023-12-30T01:22:56.066998476Z   relabel_configs:
2023-12-30T01:22:56.067002214Z   - source_labels:
2023-12-30T01:22:56.067005869Z     - __address__
2023-12-30T01:22:56.067009602Z     target_label: __param_target
2023-12-30T01:22:56.067013348Z   - source_labels:
2023-12-30T01:22:56.067016985Z     - __param_target
2023-12-30T01:22:56.067020699Z     target_label: instance
2023-12-30T01:22:56.067039103Z   - replacement: eechart-0077778702-snmpexporter:80
2023-12-30T01:22:56.067043078Z     target_label: __address__
2023-12-30T01:22:56.067046801Z   static_configs:
2023-12-30T01:22:56.067050496Z   - targets:
2023-12-30T01:22:56.067054252Z     - 172.21.249.236
2023-12-30T01:22:56.067057980Z 
2023-12-30T01:22:56.070127006Z New config written, updating prometheus
2023-12-30T01:23:01.114666904Z Prometheus config update successful
2023-12-30T01:23:01.119202640Z 
2023-12-30T01:23:20.324222849Z Change detected, updating prometheus config
2023-12-30T01:23:20.324242251Z {'_id': {'_data': '82658F7108000000082B022C0100296E5A1004AA4C5DE2B010495E9291FE6700CF3E2A46645F69640064658F70EF48EAB65B9A051C9D0004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1703899400, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('658f70ef48eab65b9a051c9d')}}
2023-12-30T01:23:20.326244514Z Jobs detected : 1
2023-12-30T01:23:20.333115452Z Writing new config file to /etc/prometheus/prometheus.yml
2023-12-30T01:23:20.335265374Z alerting:
2023-12-30T01:23:20.335278143Z   alertmanagers:
2023-12-30T01:23:20.335282913Z   - static_configs:
2023-12-30T01:23:20.335287224Z     - targets: null
2023-12-30T01:23:20.335291277Z global:
2023-12-30T01:23:20.335295368Z   evaluation_interval: 15s
2023-12-30T01:23:20.335299310Z   scrape_interval: 15s
2023-12-30T01:23:20.335318239Z rule_files: null
2023-12-30T01:23:20.335322744Z scrape_configs:
2023-12-30T01:23:20.335326551Z - job_name: mon_exporter
2023-12-30T01:23:20.335330255Z   static_configs:
2023-12-30T01:23:20.335334092Z   - targets:
2023-12-30T01:23:20.335338648Z     - mon:8000
2023-12-30T01:23:20.335342490Z 
2023-12-30T01:23:20.337118349Z New config written, updating prometheus
2023-12-30T01:23:25.362320184Z Prometheus config update successful
2023-12-30T01:23:25.364469831Z