2024-01-10T21:30:20.665055347Z level=info ts=2024-01-10T21:30:20.664Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d
2024-01-10T21:30:20.665094387Z level=info ts=2024-01-10T21:30:20.664Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)"
2024-01-10T21:30:20.665099131Z level=info ts=2024-01-10T21:30:20.664Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)"
2024-01-10T21:30:20.665113137Z level=info ts=2024-01-10T21:30:20.665Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))"
2024-01-10T21:30:20.665116795Z level=info ts=2024-01-10T21:30:20.665Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)"
2024-01-10T21:30:20.665127419Z level=info ts=2024-01-10T21:30:20.665Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)"
2024-01-10T21:30:20.667529141Z level=info ts=2024-01-10T21:30:20.667Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-01-10T21:30:20.670986093Z level=info ts=2024-01-10T21:30:20.670Z caller=main.go:824 msg="Starting TSDB ..."
2024-01-10T21:30:20.672924500Z level=info ts=2024-01-10T21:30:20.672Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false
2024-01-10T21:30:20.716598868Z level=info ts=2024-01-10T21:30:20.716Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-01-10T21:30:20.716611682Z level=info ts=2024-01-10T21:30:20.716Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.733µs
2024-01-10T21:30:20.716616549Z level=info ts=2024-01-10T21:30:20.716Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while"
2024-01-10T21:30:20.718061903Z level=info ts=2024-01-10T21:30:20.717Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-01-10T21:30:20.718074628Z level=info ts=2024-01-10T21:30:20.717Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=180.378µs wal_replay_duration=1.299419ms total_replay_duration=1.503517ms
2024-01-10T21:30:20.730193834Z level=info ts=2024-01-10T21:30:20.730Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC
2024-01-10T21:30:20.730207415Z level=info ts=2024-01-10T21:30:20.730Z caller=main.go:854 msg="TSDB started"
2024-01-10T21:30:20.730210960Z level=info ts=2024-01-10T21:30:20.730Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-10T21:30:20.731607970Z level=info ts=2024-01-10T21:30:20.731Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.118327ms remote_storage=9.215µs web_handler=675ns query_engine=3.799µs scrape=277.334µs scrape_sd=36.639µs notify=33.356µs notify_sd=10.812µs rules=4.62µs
2024-01-10T21:30:20.731620976Z level=info ts=2024-01-10T21:30:20.731Z caller=main.go:796 msg="Server is ready to receive web requests."
2024-01-10T21:34:57.588332335Z level=info ts=2024-01-10T21:34:57.587Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-10T21:34:57.589641789Z level=info ts=2024-01-10T21:34:57.589Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.955303ms remote_storage=8.542µs web_handler=661ns query_engine=3.785µs scrape=140.123µs scrape_sd=41.865µs notify=25.422µs notify_sd=9.631µs rules=46.947µs
2024-01-10T21:35:07.657519781Z level=info ts=2024-01-10T21:35:07.657Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-10T21:35:07.658652965Z level=info ts=2024-01-10T21:35:07.658Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.123715ms remote_storage=3.65µs web_handler=1.029µs query_engine=1.844µs scrape=197.061µs scrape_sd=61.236µs notify=19.249µs notify_sd=12.762µs rules=1.674µs
2024-01-10T21:35:17.716548957Z level=info ts=2024-01-10T21:35:17.716Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-10T21:35:17.717376533Z level=info ts=2024-01-10T21:35:17.717Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=956.516µs remote_storage=4.241µs web_handler=1.001µs query_engine=1.909µs scrape=186.448µs scrape_sd=60.174µs notify=30.952µs notify_sd=24.532µs rules=2.559µs
2024-01-11T00:30:54.206105101Z level=info ts=2024-01-11T00:30:54.205Z caller=compact.go:518 component=tsdb msg="write block" mint=1704922239126 maxt=1704924000000 ulid=01HKTYVNNX7A3XR2ACZ4CVFY4C duration=64.241132ms
2024-01-11T00:30:54.210731155Z level=info ts=2024-01-11T00:30:54.210Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=3.675385ms
2024-01-11T01:00:24.181277831Z level=info ts=2024-01-11T01:00:24.180Z caller=compact.go:518 component=tsdb msg="write block" mint=1704924009129 maxt=1704931200000 ulid=01HKV0HP6537TZ9D811P31STCW duration=46.976097ms
2024-01-11T01:00:24.183540880Z level=info ts=2024-01-11T01:00:24.183Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.814712ms
2024-01-11T01:04:24.193396821Z level=info ts=2024-01-11T01:04:24.192Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-11T01:04:24.197782173Z level=info ts=2024-01-11T01:04:24.197Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.853348ms remote_storage=18.777µs web_handler=2.251µs query_engine=9.805µs scrape=389.887µs scrape_sd=208.335µs notify=242.649µs notify_sd=32.219µs rules=17.483µs
2024-01-11T01:04:47.877658611Z level=info ts=2024-01-11T01:04:47.877Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-01-11T01:04:47.878236979Z level=info ts=2024-01-11T01:04:47.878Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=594.516µs remote_storage=2.1µs web_handler=438ns query_engine=1.06µs scrape=170.36µs scrape_sd=24.803µs notify=10.975µs notify_sd=10.337µs rules=2.116µs
2024-01-10T21:34:35.380989815Z Created MongoClient to connect to MongoDB!
2024-01-10T21:34:35.381079890Z Refreshing prometheus config file for first time
2024-01-10T21:34:57.544568661Z Jobs detected : 0
2024-01-10T21:34:57.556892408Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-10T21:34:57.570289876Z alerting:
2024-01-10T21:34:57.570306639Z   alertmanagers:
2024-01-10T21:34:57.570310294Z   - static_configs:
2024-01-10T21:34:57.570313463Z     - targets: null
2024-01-10T21:34:57.570316494Z global:
2024-01-10T21:34:57.570320513Z   evaluation_interval: 15s
2024-01-10T21:34:57.570324646Z   scrape_interval: 15s
2024-01-10T21:34:57.570328714Z rule_files: null
2024-01-10T21:34:57.570332652Z scrape_configs:
2024-01-10T21:34:57.570336758Z - job_name: mon_exporter
2024-01-10T21:34:57.570340535Z   static_configs:
2024-01-10T21:34:57.570343432Z   - targets:
2024-01-10T21:34:57.570346518Z     - mon:8000
2024-01-10T21:34:57.578336959Z 
2024-01-10T21:34:57.580775822Z New config written, updating prometheus
2024-01-10T21:35:02.620343312Z Prometheus config update successful
2024-01-10T21:35:07.625464871Z Refreshing prometheus config file for first time
2024-01-10T21:35:07.629450478Z Jobs detected : 1
2024-01-10T21:35:07.644340008Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-10T21:35:07.647937665Z alerting:
2024-01-10T21:35:07.647977564Z   alertmanagers:
2024-01-10T21:35:07.647987135Z   - static_configs:
2024-01-10T21:35:07.647995607Z     - targets: null
2024-01-10T21:35:07.648003805Z global:
2024-01-10T21:35:07.648011886Z   evaluation_interval: 15s
2024-01-10T21:35:07.648020006Z   scrape_interval: 15s
2024-01-10T21:35:07.648027877Z rule_files: null
2024-01-10T21:35:07.648036464Z scrape_configs:
2024-01-10T21:35:07.648044533Z - job_name: mon_exporter
2024-01-10T21:35:07.648077822Z   static_configs:
2024-01-10T21:35:07.648086976Z   - targets:
2024-01-10T21:35:07.648094990Z     - mon:8000
2024-01-10T21:35:07.648106163Z 
2024-01-10T21:35:07.651985753Z New config written, updating prometheus
2024-01-10T21:35:12.683513915Z Prometheus config update successful
2024-01-10T21:35:17.688501070Z Refreshing prometheus config file for first time
2024-01-10T21:35:17.694145257Z Jobs detected : 1
2024-01-10T21:35:17.706087978Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-10T21:35:17.708653970Z alerting:
2024-01-10T21:35:17.708690499Z   alertmanagers:
2024-01-10T21:35:17.708768708Z   - static_configs:
2024-01-10T21:35:17.708778961Z     - targets: null
2024-01-10T21:35:17.708786877Z global:
2024-01-10T21:35:17.708823572Z   evaluation_interval: 15s
2024-01-10T21:35:17.708832468Z   scrape_interval: 15s
2024-01-10T21:35:17.708840275Z rule_files: null
2024-01-10T21:35:17.708848003Z scrape_configs:
2024-01-10T21:35:17.708855577Z - job_name: mon_exporter
2024-01-10T21:35:17.708863433Z   static_configs:
2024-01-10T21:35:17.708871288Z   - targets:
2024-01-10T21:35:17.708906692Z     - mon:8000
2024-01-10T21:35:17.708916195Z 
2024-01-10T21:35:17.712490741Z New config written, updating prometheus
2024-01-10T21:35:22.746700375Z Prometheus config update successful
2024-01-10T21:35:27.759057960Z Listening to changes in prometheus jobs collection
2024-01-11T01:04:24.161559575Z Change detected, updating prometheus config
2024-01-11T01:04:24.162194552Z {'_id': {'_data': '82659F3E98000000032B022C0100296E5A1004239648E6C1294E88B4F1E10460D8610E46645F69640064659F3E98EDEF2C2188C3D1790004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1704935064, 3), 'fullDocument': {'_id': ObjectId('659f3e98edef2c2188c3d179'), 'job_name': '23c98325599c47229098f78836c89f38', 'metrics_path': '/snmp', 'nsr_id': 'e6d49650-ac60-42f0-8866-b26a10e8d09b', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0058068852-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.248.172']}], 'vnfr_id': '23c98325599c47229098f78836c89f38'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('659f3e98edef2c2188c3d179')}}
2024-01-11T01:04:24.166893395Z Jobs detected : 2
2024-01-11T01:04:24.177502219Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-11T01:04:24.182589716Z alerting:
2024-01-11T01:04:24.182656577Z   alertmanagers:
2024-01-11T01:04:24.182668080Z   - static_configs:
2024-01-11T01:04:24.182677729Z     - targets: null
2024-01-11T01:04:24.182686735Z global:
2024-01-11T01:04:24.182724157Z   evaluation_interval: 15s
2024-01-11T01:04:24.182734770Z   scrape_interval: 15s
2024-01-11T01:04:24.182743616Z rule_files: null
2024-01-11T01:04:24.182752327Z scrape_configs:
2024-01-11T01:04:24.182760848Z - job_name: mon_exporter
2024-01-11T01:04:24.182769584Z   static_configs:
2024-01-11T01:04:24.182778236Z   - targets:
2024-01-11T01:04:24.182812690Z     - mon:8000
2024-01-11T01:04:24.182822203Z - job_name: 23c98325599c47229098f78836c89f38
2024-01-11T01:04:24.182831414Z   metrics_path: /snmp
2024-01-11T01:04:24.182840101Z   params:
2024-01-11T01:04:24.182907151Z     module:
2024-01-11T01:04:24.182916481Z     - osm-snmp
2024-01-11T01:04:24.182924110Z   relabel_configs:
2024-01-11T01:04:24.182931722Z   - source_labels:
2024-01-11T01:04:24.182939287Z     - __address__
2024-01-11T01:04:24.182946966Z     target_label: __param_target
2024-01-11T01:04:24.182985243Z   - source_labels:
2024-01-11T01:04:24.182993048Z     - __param_target
2024-01-11T01:04:24.183000576Z     target_label: instance
2024-01-11T01:04:24.183008271Z   - replacement: eechart-0058068852-snmpexporter:80
2024-01-11T01:04:24.183016028Z     target_label: __address__
2024-01-11T01:04:24.183023657Z   static_configs:
2024-01-11T01:04:24.183031136Z   - targets:
2024-01-11T01:04:24.183062978Z     - 172.21.248.172
2024-01-11T01:04:24.183071836Z 
2024-01-11T01:04:24.186628135Z New config written, updating prometheus
2024-01-11T01:04:29.231119592Z Prometheus config update successful
2024-01-11T01:04:29.236032455Z 
2024-01-11T01:04:47.866779792Z Change detected, updating prometheus config
2024-01-11T01:04:47.866811170Z {'_id': {'_data': '82659F3EAF000000082B022C0100296E5A1004239648E6C1294E88B4F1E10460D8610E46645F69640064659F3E98EDEF2C2188C3D1790004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1704935087, 8), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('659f3e98edef2c2188c3d179')}}
2024-01-11T01:04:47.867732925Z Jobs detected : 1
2024-01-11T01:04:47.871910909Z Writing new config file to /etc/prometheus/prometheus.yml
2024-01-11T01:04:47.873200657Z alerting:
2024-01-11T01:04:47.873210022Z   alertmanagers:
2024-01-11T01:04:47.873214116Z   - static_configs:
2024-01-11T01:04:47.873218052Z     - targets: null
2024-01-11T01:04:47.873221745Z global:
2024-01-11T01:04:47.873225434Z   evaluation_interval: 15s
2024-01-11T01:04:47.873238441Z   scrape_interval: 15s
2024-01-11T01:04:47.873242475Z rule_files: null
2024-01-11T01:04:47.873246331Z scrape_configs:
2024-01-11T01:04:47.873250028Z - job_name: mon_exporter
2024-01-11T01:04:47.873253799Z   static_configs:
2024-01-11T01:04:47.873257496Z   - targets:
2024-01-11T01:04:47.873261187Z     - mon:8000
2024-01-11T01:04:47.873264886Z 
2024-01-11T01:04:47.874246686Z New config written, updating prometheus
2024-01-11T01:04:52.904371446Z Prometheus config update successful
2024-01-11T01:04:52.907990253Z