2024-05-09T20:44:23.714068696Z ts=2024-05-09T20:44:23.713Z caller=main.go:539 level=info msg="No time or size retention was set so using the default time retention" duration=15d
2024-05-09T20:44:23.714129485Z ts=2024-05-09T20:44:23.714Z caller=main.go:583 level=info msg="Starting Prometheus Server" mode=server version="(version=2.47.0, branch=HEAD, revision=efa34a5840661c29c2e362efa76bc3a70dccb335)"
2024-05-09T20:44:23.714210312Z ts=2024-05-09T20:44:23.714Z caller=main.go:588 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=root@409eb5e6b30c, date=20230906-10:20:07, tags=netgo,builtinassets,stringlabels)"
2024-05-09T20:44:23.714301157Z ts=2024-05-09T20:44:23.714Z caller=main.go:589 level=info host_details="(Linux 5.15.0-47-generic #51-Ubuntu SMP Thu Aug 11 07:51:15 UTC 2022 x86_64 prometheus-0 (none))"
2024-05-09T20:44:23.714417120Z ts=2024-05-09T20:44:23.714Z caller=main.go:590 level=info fd_limits="(soft=1048576, hard=1048576)"
2024-05-09T20:44:23.714507140Z ts=2024-05-09T20:44:23.714Z caller=main.go:591 level=info vm_limits="(soft=unlimited, hard=unlimited)"
2024-05-09T20:44:23.716551753Z ts=2024-05-09T20:44:23.716Z caller=web.go:566 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-05-09T20:44:23.717350131Z ts=2024-05-09T20:44:23.717Z caller=main.go:1024 level=info msg="Starting TSDB ..."
2024-05-09T20:44:23.723796877Z ts=2024-05-09T20:44:23.723Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9090
2024-05-09T20:44:23.724328649Z ts=2024-05-09T20:44:23.724Z caller=tls_config.go:277 level=info component=web msg="TLS is disabled." http2=false address=[::]:9090
2024-05-09T20:44:23.725555592Z ts=2024-05-09T20:44:23.725Z caller=head.go:600 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-05-09T20:44:23.725566664Z ts=2024-05-09T20:44:23.725Z caller=head.go:681 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.619µs
2024-05-09T20:44:23.725570004Z ts=2024-05-09T20:44:23.725Z caller=head.go:689 level=info component=tsdb msg="Replaying WAL, this may take a while"
2024-05-09T20:44:23.725989651Z ts=2024-05-09T20:44:23.725Z caller=head.go:760 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-05-09T20:44:23.725997676Z ts=2024-05-09T20:44:23.725Z caller=head.go:797 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=43.251µs wal_replay_duration=373.396µs wbl_replay_duration=160ns total_replay_duration=443.099µs
2024-05-09T20:44:23.728386988Z ts=2024-05-09T20:44:23.728Z caller=main.go:1045 level=info fs_type=EXT4_SUPER_MAGIC
2024-05-09T20:44:23.728400190Z ts=2024-05-09T20:44:23.728Z caller=main.go:1048 level=info msg="TSDB started"
2024-05-09T20:44:23.728404508Z ts=2024-05-09T20:44:23.728Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T20:44:23.729826973Z ts=2024-05-09T20:44:23.729Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.392295ms db_storage=1.376µs remote_storage=1.783µs web_handler=309ns query_engine=738ns scrape=277.758µs scrape_sd=30.196µs notify=41.548µs notify_sd=8.614µs rules=761.677µs tracing=7.446µs
2024-05-09T20:44:23.729840675Z ts=2024-05-09T20:44:23.729Z caller=main.go:1009 level=info msg="Server is ready to receive web requests."
2024-05-09T20:44:23.729844402Z ts=2024-05-09T20:44:23.729Z caller=manager.go:1009 level=info component="rule manager" msg="Starting rule manager..."
2024-05-09T20:45:25.616920391Z ts=2024-05-09T20:45:25.616Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T20:45:25.654530981Z ts=2024-05-09T20:45:25.654Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=37.711453ms db_storage=2.939µs remote_storage=2.301µs web_handler=5.486µs query_engine=2.389µs scrape=176.737µs scrape_sd=59.566µs notify=66.714µs notify_sd=9.49µs rules=16.981235ms tracing=22.274µs
2024-05-09T21:02:58.977370980Z ts=2024-05-09T21:02:58.977Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T21:02:58.980535386Z ts=2024-05-09T21:02:58.980Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.262952ms db_storage=16.971µs remote_storage=2.149µs web_handler=5.047µs query_engine=1.153µs scrape=198.768µs scrape_sd=52.586µs notify=74.134µs notify_sd=7.873µs rules=2.021348ms tracing=17.139µs
2024-05-09T21:03:04.031354934Z ts=2024-05-09T21:03:04.031Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T21:03:04.033177999Z ts=2024-05-09T21:03:04.033Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.806541ms db_storage=1.503µs remote_storage=2.278µs web_handler=421ns query_engine=1.155µs scrape=103.915µs scrape_sd=74.458µs notify=18.762µs notify_sd=14.869µs rules=1.222222ms tracing=6.95µs
2024-05-09T21:14:46.541403422Z ts=2024-05-09T21:14:46.541Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T21:14:46.546260157Z ts=2024-05-09T21:14:46.546Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.864502ms db_storage=2.256µs remote_storage=9.51µs web_handler=7.399µs query_engine=2.124µs scrape=227.326µs scrape_sd=54.45µs notify=71.74µs notify_sd=11.315µs rules=3.446531ms tracing=23.038µs
2024-05-09T21:14:51.594134257Z ts=2024-05-09T21:14:51.594Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-09T21:14:51.595754027Z ts=2024-05-09T21:14:51.595Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.580741ms db_storage=1.417µs remote_storage=1.433µs web_handler=597ns query_engine=1.084µs scrape=92.541µs scrape_sd=23.615µs notify=31.104µs notify_sd=8.006µs rules=1.066473ms tracing=4.853µs
2024-05-09T20:45:20.450500581Z Created MongoClient to connect to MongoDB!
2024-05-09T20:45:20.450529376Z Generating prometheus config files
2024-05-09T20:45:20.518693596Z Jobs detected: 0
2024-05-09T20:45:20.528947038Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T20:45:20.528965405Z alerting:
2024-05-09T20:45:20.528968807Z   alertmanagers:
2024-05-09T20:45:20.528972738Z   - static_configs:
2024-05-09T20:45:20.528975756Z     - targets:
2024-05-09T20:45:20.528979193Z       - alertmanager:9093
2024-05-09T20:45:20.528982129Z global:
2024-05-09T20:45:20.528985161Z   evaluation_interval: 15s
2024-05-09T20:45:20.528988053Z   scrape_interval: 15s
2024-05-09T20:45:20.528991009Z rule_files:
2024-05-09T20:45:20.528993928Z - osm_metric_rules.yml
2024-05-09T20:45:20.528996904Z - osm_alert_rules.yml
2024-05-09T20:45:20.528999880Z scrape_configs:
2024-05-09T20:45:20.529002842Z - job_name: mon_exporter
2024-05-09T20:45:20.529005723Z   static_configs:
2024-05-09T20:45:20.529008610Z   - targets:
2024-05-09T20:45:20.529011550Z     - mon:8000
2024-05-09T20:45:20.529014448Z - honor_labels: true
2024-05-09T20:45:20.529017345Z   job_name: pushgateway
2024-05-09T20:45:20.529020235Z   scrape_interval: 30s
2024-05-09T20:45:20.529023050Z   static_configs:
2024-05-09T20:45:20.529026296Z   - targets:
2024-05-09T20:45:20.529029777Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T20:45:20.529032539Z 
2024-05-09T20:45:20.529192630Z Alerts read: 0
2024-05-09T20:45:20.530891525Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T20:45:20.532168927Z groups:
2024-05-09T20:45:20.532177733Z - name: osm_alert_rules
2024-05-09T20:45:20.532180894Z   rules:
2024-05-09T20:45:20.532183883Z   - alert: vdu_down
2024-05-09T20:45:20.532186779Z     annotations:
2024-05-09T20:45:20.532190416Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T20:45:20.532194107Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T20:45:20.532197043Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T20:45:20.532200691Z     expr: vm_status_extended != 1
2024-05-09T20:45:20.532203629Z     for: 3m
2024-05-09T20:45:20.532206400Z 
2024-05-09T20:45:20.533020050Z New config written, updating prometheus
2024-05-09T20:45:20.540728612Z Error in configuration attempt! Number of tries: 1/3
2024-05-09T20:45:20.540740304Z Cannot connect to host prometheus:9090 ssl:default [Connect call failed ('10.99.244.230', 9090)]
2024-05-09T20:45:25.544896764Z Generating prometheus config files
2024-05-09T20:45:25.564557329Z Jobs detected: 0
2024-05-09T20:45:25.564572050Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T20:45:25.573985104Z alerting:
2024-05-09T20:45:25.574002451Z   alertmanagers:
2024-05-09T20:45:25.574006518Z   - static_configs:
2024-05-09T20:45:25.574009958Z     - targets:
2024-05-09T20:45:25.574013328Z       - alertmanager:9093
2024-05-09T20:45:25.574016342Z global:
2024-05-09T20:45:25.574019433Z   evaluation_interval: 15s
2024-05-09T20:45:25.574022643Z   scrape_interval: 15s
2024-05-09T20:45:25.574025966Z rule_files:
2024-05-09T20:45:25.574029070Z - osm_metric_rules.yml
2024-05-09T20:45:25.574032196Z - osm_alert_rules.yml
2024-05-09T20:45:25.574035191Z scrape_configs:
2024-05-09T20:45:25.574038128Z - job_name: mon_exporter
2024-05-09T20:45:25.574041066Z   static_configs:
2024-05-09T20:45:25.574044180Z   - targets:
2024-05-09T20:45:25.574047105Z     - mon:8000
2024-05-09T20:45:25.574050235Z - honor_labels: true
2024-05-09T20:45:25.574069308Z   job_name: pushgateway
2024-05-09T20:45:25.574072626Z   scrape_interval: 30s
2024-05-09T20:45:25.574075642Z   static_configs:
2024-05-09T20:45:25.574078639Z   - targets:
2024-05-09T20:45:25.574082305Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T20:45:25.574085351Z 
2024-05-09T20:45:25.588370202Z Alerts read: 0
2024-05-09T20:45:25.588386834Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T20:45:25.594038515Z groups:
2024-05-09T20:45:25.594053396Z - name: osm_alert_rules
2024-05-09T20:45:25.594056633Z   rules:
2024-05-09T20:45:25.594059747Z   - alert: vdu_down
2024-05-09T20:45:25.594062759Z     annotations:
2024-05-09T20:45:25.594066544Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T20:45:25.594070004Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T20:45:25.594073049Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T20:45:25.594076451Z     expr: vm_status_extended != 1
2024-05-09T20:45:25.594079467Z     for: 3m
2024-05-09T20:45:25.594082275Z 
2024-05-09T20:45:25.595024341Z New config written, updating prometheus
2024-05-09T20:45:30.677952566Z Prometheus config update successful
2024-05-09T20:45:35.699836350Z Listening to changes in prometheus jobs and alerts collections
2024-05-09T21:02:58.946886738Z Changes detected, updating prometheus config
2024-05-09T21:02:58.948751933Z Jobs detected: 0
2024-05-09T21:02:58.954424407Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T21:02:58.957528574Z alerting:
2024-05-09T21:02:58.957547938Z   alertmanagers:
2024-05-09T21:02:58.957554518Z   - static_configs:
2024-05-09T21:02:58.957560077Z     - targets:
2024-05-09T21:02:58.957578650Z       - alertmanager:9093
2024-05-09T21:02:58.957584409Z global:
2024-05-09T21:02:58.957590806Z   evaluation_interval: 15s
2024-05-09T21:02:58.957596185Z   scrape_interval: 15s
2024-05-09T21:02:58.957601491Z rule_files:
2024-05-09T21:02:58.957606738Z - osm_metric_rules.yml
2024-05-09T21:02:58.957612020Z - osm_alert_rules.yml
2024-05-09T21:02:58.957617253Z scrape_configs:
2024-05-09T21:02:58.957622567Z - job_name: mon_exporter
2024-05-09T21:02:58.957627976Z   static_configs:
2024-05-09T21:02:58.957633189Z   - targets:
2024-05-09T21:02:58.957638502Z     - mon:8000
2024-05-09T21:02:58.957643632Z - honor_labels: true
2024-05-09T21:02:58.957648844Z   job_name: pushgateway
2024-05-09T21:02:58.957654056Z   scrape_interval: 30s
2024-05-09T21:02:58.957659305Z   static_configs:
2024-05-09T21:02:58.957664476Z   - targets:
2024-05-09T21:02:58.957670454Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T21:02:58.957675614Z 
2024-05-09T21:02:58.961233610Z Alerts read: 2
2024-05-09T21:02:58.964110468Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T21:02:58.967117251Z groups:
2024-05-09T21:02:58.967131418Z - name: osm_alert_rules
2024-05-09T21:02:58.967136365Z   rules:
2024-05-09T21:02:58.967140465Z   - alert: vdu_down
2024-05-09T21:02:58.967144552Z     annotations:
2024-05-09T21:02:58.967149076Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T21:02:58.967153637Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T21:02:58.967157857Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T21:02:58.967162243Z     expr: vm_status_extended != 1
2024-05-09T21:02:58.967166350Z     for: 3m
2024-05-09T21:02:58.967185307Z - name: _osm_alert_rules_20240509210258_
2024-05-09T21:02:58.967189637Z   rules:
2024-05-09T21:02:58.967194121Z   - alert: scalein_3c8fdd42-872e-484e-b0d1-0c498553f735
2024-05-09T21:02:58.967197968Z     expr: (count (osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:02:58.967202485Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:02:58.967206317Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-05-09T21:02:58.967210195Z     for: 1m
2024-05-09T21:02:58.967214057Z     labels:
2024-05-09T21:02:58.967217956Z       ns_id: 60c5c89f-d9d5-4e29-be7c-a57f7f7598c4
2024-05-09T21:02:58.967221747Z       vdu_id: hackfest_basic_metrics-VM
2024-05-09T21:02:58.967226090Z       vnf_member_index: vnf
2024-05-09T21:02:58.967230015Z   - alert: scaleout_a4b79f58-2fd2-4dac-870f-7c15b336e329
2024-05-09T21:02:58.967233875Z     expr: (count (osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:02:58.967237764Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:02:58.967241569Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-05-09T21:02:58.967245407Z     for: 1m
2024-05-09T21:02:58.967249252Z     labels:
2024-05-09T21:02:58.967253044Z       ns_id: 60c5c89f-d9d5-4e29-be7c-a57f7f7598c4
2024-05-09T21:02:58.967257230Z       vdu_id: hackfest_basic_metrics-VM
2024-05-09T21:02:58.967261253Z       vnf_member_index: vnf
2024-05-09T21:02:58.967264916Z 
2024-05-09T21:02:58.970163525Z New config written, updating prometheus
2024-05-09T21:03:04.000157605Z Prometheus config update successful
2024-05-09T21:03:04.000185695Z 
2024-05-09T21:03:04.001489493Z Changes detected, updating prometheus config
2024-05-09T21:03:04.006727695Z Jobs detected: 0
2024-05-09T21:03:04.012145825Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T21:03:04.013992425Z alerting:
2024-05-09T21:03:04.014009242Z   alertmanagers:
2024-05-09T21:03:04.014015186Z   - static_configs:
2024-05-09T21:03:04.014020126Z     - targets:
2024-05-09T21:03:04.014025335Z       - alertmanager:9093
2024-05-09T21:03:04.014030026Z global:
2024-05-09T21:03:04.014035032Z   evaluation_interval: 15s
2024-05-09T21:03:04.014039727Z   scrape_interval: 15s
2024-05-09T21:03:04.014044382Z rule_files:
2024-05-09T21:03:04.014049049Z - osm_metric_rules.yml
2024-05-09T21:03:04.014053630Z - osm_alert_rules.yml
2024-05-09T21:03:04.014058348Z scrape_configs:
2024-05-09T21:03:04.014062937Z - job_name: mon_exporter
2024-05-09T21:03:04.014067638Z   static_configs:
2024-05-09T21:03:04.014072360Z   - targets:
2024-05-09T21:03:04.014077045Z     - mon:8000
2024-05-09T21:03:04.014081646Z - honor_labels: true
2024-05-09T21:03:04.014086337Z   job_name: pushgateway
2024-05-09T21:03:04.014090887Z   scrape_interval: 30s
2024-05-09T21:03:04.014095496Z   static_configs:
2024-05-09T21:03:04.014100081Z   - targets:
2024-05-09T21:03:04.014105570Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T21:03:04.014110199Z 
2024-05-09T21:03:04.017426239Z Alerts read: 2
2024-05-09T21:03:04.019700883Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T21:03:04.023222081Z groups:
2024-05-09T21:03:04.023235755Z - name: osm_alert_rules
2024-05-09T21:03:04.023240837Z   rules:
2024-05-09T21:03:04.023245502Z   - alert: vdu_down
2024-05-09T21:03:04.023262930Z     annotations:
2024-05-09T21:03:04.023268617Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T21:03:04.023273508Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T21:03:04.023277786Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T21:03:04.023282698Z     expr: vm_status_extended != 1
2024-05-09T21:03:04.023287171Z     for: 3m
2024-05-09T21:03:04.023291625Z - name: _osm_alert_rules_20240509210304_
2024-05-09T21:03:04.023295870Z   rules:
2024-05-09T21:03:04.023300601Z   - alert: scalein_3c8fdd42-872e-484e-b0d1-0c498553f735
2024-05-09T21:03:04.023305338Z     expr: (count (osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:03:04.023310246Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:03:04.023314681Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-05-09T21:03:04.023319052Z     for: 1m
2024-05-09T21:03:04.023323217Z     labels:
2024-05-09T21:03:04.023327508Z       ns_id: 60c5c89f-d9d5-4e29-be7c-a57f7f7598c4
2024-05-09T21:03:04.023331679Z       vdu_id: hackfest_basic_metrics-VM
2024-05-09T21:03:04.023336466Z       vnf_member_index: vnf
2024-05-09T21:03:04.023340692Z   - alert: scaleout_a4b79f58-2fd2-4dac-870f-7c15b336e329
2024-05-09T21:03:04.023344943Z     expr: (count (osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:03:04.023349462Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="60c5c89f-d9d5-4e29-be7c-a57f7f7598c4",
2024-05-09T21:03:04.023353722Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-05-09T21:03:04.023357955Z     for: 1m
2024-05-09T21:03:04.023362218Z     labels:
2024-05-09T21:03:04.023366436Z       ns_id: 60c5c89f-d9d5-4e29-be7c-a57f7f7598c4
2024-05-09T21:03:04.023370944Z       vdu_id: hackfest_basic_metrics-VM
2024-05-09T21:03:04.023375196Z       vnf_member_index: vnf
2024-05-09T21:03:04.023379220Z 
2024-05-09T21:03:04.026509955Z New config written, updating prometheus
2024-05-09T21:03:09.060254496Z Prometheus config update successful
2024-05-09T21:03:09.060462844Z 
2024-05-09T21:14:46.515414467Z Changes detected, updating prometheus config
2024-05-09T21:14:46.520830517Z Jobs detected: 0
2024-05-09T21:14:46.526437993Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T21:14:46.528886592Z alerting:
2024-05-09T21:14:46.528897632Z   alertmanagers:
2024-05-09T21:14:46.528901563Z   - static_configs:
2024-05-09T21:14:46.528905323Z     - targets:
2024-05-09T21:14:46.528909053Z       - alertmanager:9093
2024-05-09T21:14:46.528912310Z global:
2024-05-09T21:14:46.528915485Z   evaluation_interval: 15s
2024-05-09T21:14:46.528918514Z   scrape_interval: 15s
2024-05-09T21:14:46.528921572Z rule_files:
2024-05-09T21:14:46.528924563Z - osm_metric_rules.yml
2024-05-09T21:14:46.528927523Z - osm_alert_rules.yml
2024-05-09T21:14:46.528930691Z scrape_configs:
2024-05-09T21:14:46.528933828Z - job_name: mon_exporter
2024-05-09T21:14:46.528936916Z   static_configs:
2024-05-09T21:14:46.528939941Z   - targets:
2024-05-09T21:14:46.528942886Z     - mon:8000
2024-05-09T21:14:46.528945775Z - honor_labels: true
2024-05-09T21:14:46.528948857Z   job_name: pushgateway
2024-05-09T21:14:46.528952112Z   scrape_interval: 30s
2024-05-09T21:14:46.528955001Z   static_configs:
2024-05-09T21:14:46.528957910Z   - targets:
2024-05-09T21:14:46.528974463Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T21:14:46.528977437Z 
2024-05-09T21:14:46.532194227Z Alerts read: 0
2024-05-09T21:14:46.534732352Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T21:14:46.535996691Z groups:
2024-05-09T21:14:46.536007558Z - name: osm_alert_rules
2024-05-09T21:14:46.536010624Z   rules:
2024-05-09T21:14:46.536013471Z   - alert: vdu_down
2024-05-09T21:14:46.536016373Z     annotations:
2024-05-09T21:14:46.536019724Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T21:14:46.536025532Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T21:14:46.536028429Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T21:14:46.536031932Z     expr: vm_status_extended != 1
2024-05-09T21:14:46.536034794Z     for: 3m
2024-05-09T21:14:46.536037431Z 
2024-05-09T21:14:46.537174676Z New config written, updating prometheus
2024-05-09T21:14:51.563157557Z Prometheus config update successful
2024-05-09T21:14:51.563176077Z 
2024-05-09T21:14:51.563179534Z Changes detected, updating prometheus config
2024-05-09T21:14:51.573560157Z Jobs detected: 0
2024-05-09T21:14:51.579084401Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-09T21:14:51.581157624Z alerting:
2024-05-09T21:14:51.581170011Z   alertmanagers:
2024-05-09T21:14:51.581174015Z   - static_configs:
2024-05-09T21:14:51.581189781Z     - targets:
2024-05-09T21:14:51.581193633Z       - alertmanager:9093
2024-05-09T21:14:51.581207934Z global:
2024-05-09T21:14:51.581211286Z   evaluation_interval: 15s
2024-05-09T21:14:51.581218586Z   scrape_interval: 15s
2024-05-09T21:14:51.581221664Z rule_files:
2024-05-09T21:14:51.581224742Z - osm_metric_rules.yml
2024-05-09T21:14:51.581238788Z - osm_alert_rules.yml
2024-05-09T21:14:51.581241845Z scrape_configs:
2024-05-09T21:14:51.581244844Z - job_name: mon_exporter
2024-05-09T21:14:51.581247536Z   static_configs:
2024-05-09T21:14:51.581250250Z   - targets:
2024-05-09T21:14:51.581253118Z     - mon:8000
2024-05-09T21:14:51.581255882Z - honor_labels: true
2024-05-09T21:14:51.581258623Z   job_name: pushgateway
2024-05-09T21:14:51.581261433Z   scrape_interval: 30s
2024-05-09T21:14:51.581264154Z   static_configs:
2024-05-09T21:14:51.581266898Z   - targets:
2024-05-09T21:14:51.581270322Z     - pushgateway-prometheus-pushgateway:9091
2024-05-09T21:14:51.581272990Z 
2024-05-09T21:14:51.585606056Z Alerts read: 0
2024-05-09T21:14:51.588013016Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-09T21:14:51.589411714Z groups:
2024-05-09T21:14:51.589422393Z - name: osm_alert_rules
2024-05-09T21:14:51.589425884Z   rules:
2024-05-09T21:14:51.589429283Z   - alert: vdu_down
2024-05-09T21:14:51.589432270Z     annotations:
2024-05-09T21:14:51.589435826Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-09T21:14:51.589439358Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-09T21:14:51.589442555Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-09T21:14:51.589446055Z     expr: vm_status_extended != 1
2024-05-09T21:14:51.589449053Z     for: 3m
2024-05-09T21:14:51.589451837Z 
2024-05-09T21:14:51.590450520Z New config written, updating prometheus
2024-05-09T21:14:56.621294481Z Prometheus config update successful
2024-05-09T21:14:56.621310836Z