2024-06-24T17:44:06.093154896Z level=info ts=2024-06-24T17:44:06.092977809Z caller=main.go:137 msg="Starting prometheus-config-reloader" version="(version=0.71.2, branch=refs/tags/v0.71.2, revision=af2014407bdc25c4fc2b26cd99c9655235ebdf88)"
2024-06-24T17:44:06.093189588Z level=info ts=2024-06-24T17:44:06.093049671Z caller=main.go:138 build_context="(go=go1.21.6, platform=linux/amd64, user=Action-Run-ID-7656327832, date=20240125-14:58:54, tags=unknown)"
2024-06-24T17:44:06.094360025Z level=info ts=2024-06-24T17:44:06.093522416Z caller=reloader.go:246 msg="reloading via HTTP"
2024-06-24T17:44:06.094379735Z level=info ts=2024-06-24T17:44:06.093588491Z caller=reloader.go:282 msg="started watching config file and directories for changes" cfg= out= dirs=/etc/config
2024-06-24T17:47:06.109432387Z level=info ts=2024-06-24T17:47:06.10874839Z caller=reloader.go:424 msg="Reload triggered" cfg_in= cfg_out= watched_dirs=/etc/config
2024-06-24T17:45:02.394420174Z ts=2024-06-24T17:45:02.394Z caller=main.go:588 level=info msg="Starting Prometheus Server" mode=server version="(version=2.49.1, branch=HEAD, revision=43e14844a33b65e2a396e3944272af8b3a494071)"
2024-06-24T17:45:02.394452989Z ts=2024-06-24T17:45:02.394Z caller=main.go:593 level=info build_context="(go=go1.21.6, platform=linux/amd64, user=root@6d5f4c649d25, date=20240115-16:58:43, tags=netgo,builtinassets,stringlabels)"
2024-06-24T17:45:02.394463495Z ts=2024-06-24T17:45:02.394Z caller=main.go:594 level=info host_details="(Linux 5.15.0-47-generic #51-Ubuntu SMP Thu Aug 11 07:51:15 UTC 2022 x86_64 prometheus-0 (none))"
2024-06-24T17:45:02.394493034Z ts=2024-06-24T17:45:02.394Z caller=main.go:595 level=info fd_limits="(soft=1048576, hard=1048576)"
2024-06-24T17:45:02.394540588Z ts=2024-06-24T17:45:02.394Z caller=main.go:596 level=info vm_limits="(soft=unlimited, hard=unlimited)"
2024-06-24T17:45:02.398965634Z ts=2024-06-24T17:45:02.398Z caller=web.go:565 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-06-24T17:45:02.399963165Z ts=2024-06-24T17:45:02.399Z caller=main.go:1039 level=info msg="Starting TSDB ..."
2024-06-24T17:45:02.416058868Z ts=2024-06-24T17:45:02.415Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9090
2024-06-24T17:45:02.416072824Z ts=2024-06-24T17:45:02.415Z caller=tls_config.go:277 level=info component=web msg="TLS is disabled." http2=false address=[::]:9090
2024-06-24T17:45:02.418100226Z ts=2024-06-24T17:45:02.417Z caller=head.go:606 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-06-24T17:45:02.418237698Z ts=2024-06-24T17:45:02.418Z caller=head.go:687 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.698µs
2024-06-24T17:45:02.418255657Z ts=2024-06-24T17:45:02.418Z caller=head.go:695 level=info component=tsdb msg="Replaying WAL, this may take a while"
2024-06-24T17:45:02.418862276Z ts=2024-06-24T17:45:02.418Z caller=head.go:766 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-06-24T17:45:02.418981238Z ts=2024-06-24T17:45:02.418Z caller=head.go:803 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=124.79µs wal_replay_duration=504.955µs wbl_replay_duration=212ns total_replay_duration=765.164µs
2024-06-24T17:45:02.422643956Z ts=2024-06-24T17:45:02.422Z caller=main.go:1060 level=info fs_type=EXT4_SUPER_MAGIC
2024-06-24T17:45:02.422793677Z ts=2024-06-24T17:45:02.422Z caller=main.go:1063 level=info msg="TSDB started"
2024-06-24T17:45:02.422911763Z ts=2024-06-24T17:45:02.422Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T17:45:02.427367730Z ts=2024-06-24T17:45:02.427Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.383727ms db_storage=1.192µs remote_storage=1.896µs web_handler=473ns query_engine=1.314µs scrape=982.895µs scrape_sd=76.05µs notify=26.346µs notify_sd=11.467µs rules=2.531449ms tracing=7.709µs
2024-06-24T17:45:02.427515616Z ts=2024-06-24T17:45:02.427Z caller=main.go:1024 level=info msg="Server is ready to receive web requests."
2024-06-24T17:45:02.427950723Z ts=2024-06-24T17:45:02.427Z caller=manager.go:146 level=info component="rule manager" msg="Starting rule manager..."
2024-06-24T17:46:34.949389299Z ts=2024-06-24T17:46:34.949Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T17:46:34.951778191Z ts=2024-06-24T17:46:34.951Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.426007ms db_storage=1.857µs remote_storage=1.942µs web_handler=5.274µs query_engine=1.185µs scrape=222.933µs scrape_sd=78.208µs notify=51.063µs notify_sd=23.127µs rules=1.325745ms tracing=5.556µs
2024-06-24T17:47:06.106276587Z ts=2024-06-24T17:47:06.105Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T17:47:06.108374877Z ts=2024-06-24T17:47:06.108Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.416088ms db_storage=4.709µs remote_storage=2.148µs web_handler=5.719µs query_engine=3.327µs scrape=183.852µs scrape_sd=133.601µs notify=27.435µs notify_sd=10.837µs rules=1.433479ms tracing=5.033µs
2024-06-24T18:00:23.182586263Z ts=2024-06-24T18:00:23.182Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T18:00:23.187637337Z ts=2024-06-24T18:00:23.187Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=5.026632ms db_storage=7.879µs remote_storage=3.088µs web_handler=8.856µs query_engine=2.523µs scrape=331.886µs scrape_sd=109.427µs notify=70.845µs notify_sd=13.494µs rules=3.171952ms tracing=17.25µs
2024-06-24T18:00:28.253074031Z ts=2024-06-24T18:00:28.252Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T18:00:28.259034241Z ts=2024-06-24T18:00:28.258Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=5.958196ms db_storage=1.568µs remote_storage=2.393µs web_handler=591ns query_engine=1.784µs scrape=256.507µs scrape_sd=104.867µs notify=91.425µs notify_sd=56.469µs rules=4.757607ms tracing=6.407µs
2024-06-24T18:02:17.589947360Z ts=2024-06-24T18:02:17.589Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"UNKNOWN\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:02:32.590098118Z ts=2024-06-24T18:02:32.589Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"UNKNOWN\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:12:17.589449484Z ts=2024-06-24T18:12:17.589Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:12:32.588739708Z ts=2024-06-24T18:12:32.588Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:12:47.589213316Z ts=2024-06-24T18:12:47.588Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:13:02.591895460Z ts=2024-06-24T18:13:02.591Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:13:17.591252282Z ts=2024-06-24T18:13:17.590Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:13:32.588745436Z ts=2024-06-24T18:13:32.588Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:13:47.588873486Z ts=2024-06-24T18:13:47.588Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:14:02.588736866Z ts=2024-06-24T18:14:02.588Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:14:17.589729125Z ts=2024-06-24T18:14:17.589Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:14:32.589715693Z ts=2024-06-24T18:14:32.589Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"34281882-02e7-4e21-be10-fe0ebddd2d3c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"02dc7f3e-c632-4b27-b418-965aa0c8ca0f\", project_id=\"ad18b5e3-8c61-4432-ba81-abf99eadd835\", vdu_id=\"3218988e-62ae-4a4d-8f8a-fce5a358a0b2\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"265f8237-fe16-40f7-9b1d-cacfe2b79727\", vm_id=\"79950747-587f-4883-ba2b-4b06352f8a41\", vnf_id=\"71d085db-1627-469f-9406-73751798755f\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-06-24T18:15:54.667489164Z ts=2024-06-24T18:15:54.667Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T18:15:54.670092705Z ts=2024-06-24T18:15:54.669Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.628734ms db_storage=2.262µs remote_storage=2.441µs web_handler=5.495µs query_engine=1.366µs scrape=226.156µs scrape_sd=79.266µs notify=42.355µs notify_sd=9.727µs rules=1.492799ms tracing=35.402µs
2024-06-24T18:15:59.733824440Z ts=2024-06-24T18:15:59.733Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-06-24T18:15:59.737657520Z ts=2024-06-24T18:15:59.737Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.686558ms db_storage=1.822µs remote_storage=2.197µs web_handler=634ns query_engine=1.712µs scrape=129.631µs scrape_sd=70.75µs notify=25.443µs notify_sd=9.218µs rules=2.99934ms tracing=7.417µs
2024-06-24T17:46:34.904739461Z Created MongoClient to connect to MongoDB!
2024-06-24T17:46:34.904771225Z Generating prometheus config files
2024-06-24T17:46:34.930174829Z Jobs detected: 0
2024-06-24T17:46:34.934357882Z Writing new config file to /etc/prometheus/prometheus.yml
2024-06-24T17:46:34.936745623Z alerting:
2024-06-24T17:46:34.936758451Z   alertmanagers:
2024-06-24T17:46:34.936762149Z   - static_configs:
2024-06-24T17:46:34.936765116Z     - targets:
2024-06-24T17:46:34.936779928Z       - alertmanager:9093
2024-06-24T17:46:34.936783315Z global:
2024-06-24T17:46:34.936786616Z   evaluation_interval: 15s
2024-06-24T17:46:34.936793543Z   scrape_interval: 15s
2024-06-24T17:46:34.936796638Z rule_files:
2024-06-24T17:46:34.936799437Z - osm_metric_rules.yml
2024-06-24T17:46:34.936802360Z - osm_alert_rules.yml
2024-06-24T17:46:34.936805339Z scrape_configs:
2024-06-24T17:46:34.936808218Z - job_name: mon_exporter
2024-06-24T17:46:34.936811130Z   static_configs:
2024-06-24T17:46:34.936814109Z   - targets:
2024-06-24T17:46:34.936817021Z     - mon:8000
2024-06-24T17:46:34.936820585Z - honor_labels: true
2024-06-24T17:46:34.936823479Z   job_name: pushgateway
2024-06-24T17:46:34.936826335Z   scrape_interval: 30s
2024-06-24T17:46:34.936829191Z   static_configs:
2024-06-24T17:46:34.936832035Z   - targets:
2024-06-24T17:46:34.936835471Z     - pushgateway-prometheus-pushgateway:9091
2024-06-24T17:46:34.936838383Z - job_name: prometheus
2024-06-24T17:46:34.936841163Z   static_configs:
2024-06-24T17:46:34.936844098Z   - targets:
2024-06-24T17:46:34.936846946Z     - localhost:9090
2024-06-24T17:46:34.936849718Z - job_name: node
2024-06-24T17:46:34.936852497Z   static_configs:
2024-06-24T17:46:34.936855624Z   - targets:
2024-06-24T17:46:34.936858517Z     - prometheus-node-exporter:9100
2024-06-24T17:46:34.936861246Z 
2024-06-24T17:46:34.940490379Z Alerts read: 0
2024-06-24T17:46:34.942077982Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-06-24T17:46:34.943071389Z groups:
2024-06-24T17:46:34.943081026Z - name: osm_alert_rules
2024-06-24T17:46:34.943084268Z   rules:
2024-06-24T17:46:34.943087071Z   - alert: vdu_down
2024-06-24T17:46:34.943089876Z     annotations:
2024-06-24T17:46:34.943093153Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-06-24T17:46:34.943096553Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-06-24T17:46:34.943099276Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-06-24T17:46:34.943102837Z     expr: vm_status_extended != 1
2024-06-24T17:46:34.943105520Z     for: 3m
2024-06-24T17:46:34.943140351Z 
2024-06-24T17:46:34.944176774Z New config written, updating prometheus
2024-06-24T17:46:39.980345609Z Prometheus config update successful
2024-06-24T17:46:44.991762493Z Listening to changes in prometheus jobs and alerts collections
2024-06-24T18:00:23.140709178Z Changes detected, updating prometheus config
2024-06-24T18:00:23.142604198Z Jobs detected: 0
2024-06-24T18:00:23.149505403Z Writing new config file to /etc/prometheus/prometheus.yml
2024-06-24T18:00:23.152858360Z alerting:
2024-06-24T18:00:23.152869019Z   alertmanagers:
2024-06-24T18:00:23.152872633Z   - static_configs:
2024-06-24T18:00:23.152875658Z     - targets:
2024-06-24T18:00:23.152879572Z       - alertmanager:9093
2024-06-24T18:00:23.152882662Z global:
2024-06-24T18:00:23.152885835Z   evaluation_interval: 15s
2024-06-24T18:00:23.152889238Z   scrape_interval: 15s
2024-06-24T18:00:23.152892207Z rule_files:
2024-06-24T18:00:23.152906683Z - osm_metric_rules.yml
2024-06-24T18:00:23.152909740Z - osm_alert_rules.yml
2024-06-24T18:00:23.152912551Z scrape_configs:
2024-06-24T18:00:23.152915523Z - job_name: mon_exporter
2024-06-24T18:00:23.152918345Z   static_configs:
2024-06-24T18:00:23.152921755Z   - targets:
2024-06-24T18:00:23.152924813Z     - mon:8000
2024-06-24T18:00:23.152927788Z - honor_labels: true
2024-06-24T18:00:23.152930638Z   job_name: pushgateway
2024-06-24T18:00:23.152933591Z   scrape_interval: 30s
2024-06-24T18:00:23.152936383Z   static_configs:
2024-06-24T18:00:23.152939249Z   - targets:
2024-06-24T18:00:23.152942527Z     - pushgateway-prometheus-pushgateway:9091
2024-06-24T18:00:23.152945534Z - job_name: prometheus
2024-06-24T18:00:23.152948716Z   static_configs:
2024-06-24T18:00:23.152951549Z   - targets:
2024-06-24T18:00:23.152954430Z     - localhost:9090
2024-06-24T18:00:23.152957284Z - job_name: node
2024-06-24T18:00:23.152964114Z   static_configs:
2024-06-24T18:00:23.152967043Z   - targets:
2024-06-24T18:00:23.152969956Z     - prometheus-node-exporter:9100
2024-06-24T18:00:23.153037920Z 
2024-06-24T18:00:23.160332560Z Alerts read: 2
2024-06-24T18:00:23.162371219Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-06-24T18:00:23.166705857Z groups:
2024-06-24T18:00:23.166726378Z - name: osm_alert_rules
2024-06-24T18:00:23.166729996Z   rules:
2024-06-24T18:00:23.166732972Z   - alert: vdu_down
2024-06-24T18:00:23.166736147Z     annotations:
2024-06-24T18:00:23.166739750Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-06-24T18:00:23.166743341Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-06-24T18:00:23.166746622Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-06-24T18:00:23.166750344Z     expr: vm_status_extended != 1
2024-06-24T18:00:23.166753365Z     for: 3m
2024-06-24T18:00:23.166756296Z - name: _osm_alert_rules_20240624180023_
2024-06-24T18:00:23.166759183Z   rules:
2024-06-24T18:00:23.166762968Z   - alert: scalein_78e8af7a-85b9-4769-a672-4d18904ac3e1
2024-06-24T18:00:23.166765887Z     expr: (count (osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:23.166772060Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:23.166776047Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-06-24T18:00:23.166779390Z     for: 1m
2024-06-24T18:00:23.166782273Z     labels:
2024-06-24T18:00:23.166787814Z       ns_id: 02dc7f3e-c632-4b27-b418-965aa0c8ca0f
2024-06-24T18:00:23.166790841Z       vdu_id: hackfest_basic_metrics-VM
2024-06-24T18:00:23.166794304Z       vnf_member_index: vnf
2024-06-24T18:00:23.166797305Z   - alert: scaleout_9e672951-fc6a-4a32-a53b-f1c996b6eead
2024-06-24T18:00:23.166800346Z     expr: (count (osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:23.166803605Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:23.166806712Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-06-24T18:00:23.166809725Z     for: 1m
2024-06-24T18:00:23.166812607Z     labels:
2024-06-24T18:00:23.166815513Z       ns_id: 02dc7f3e-c632-4b27-b418-965aa0c8ca0f
2024-06-24T18:00:23.166818395Z       vdu_id: hackfest_basic_metrics-VM
2024-06-24T18:00:23.166821320Z       vnf_member_index: vnf
2024-06-24T18:00:23.166931748Z 
2024-06-24T18:00:23.170094265Z New config written, updating prometheus
2024-06-24T18:00:28.220134124Z Prometheus config update successful
2024-06-24T18:00:28.220207431Z 
2024-06-24T18:00:28.221702764Z Changes detected, updating prometheus config
2024-06-24T18:00:28.222774271Z Jobs detected: 0
2024-06-24T18:00:28.228128861Z Writing new config file to /etc/prometheus/prometheus.yml
2024-06-24T18:00:28.232622567Z alerting:
2024-06-24T18:00:28.232635059Z   alertmanagers:
2024-06-24T18:00:28.232638797Z   - static_configs:
2024-06-24T18:00:28.232641972Z     - targets:
2024-06-24T18:00:28.232647148Z       - alertmanager:9093
2024-06-24T18:00:28.232650399Z global:
2024-06-24T18:00:28.232653512Z   evaluation_interval: 15s
2024-06-24T18:00:28.232656549Z   scrape_interval: 15s
2024-06-24T18:00:28.232659620Z rule_files:
2024-06-24T18:00:28.232662531Z - osm_metric_rules.yml
2024-06-24T18:00:28.232665389Z - osm_alert_rules.yml
2024-06-24T18:00:28.232668419Z scrape_configs:
2024-06-24T18:00:28.232671317Z - job_name: mon_exporter
2024-06-24T18:00:28.232674394Z   static_configs:
2024-06-24T18:00:28.232677427Z   - targets:
2024-06-24T18:00:28.232680348Z     - mon:8000
2024-06-24T18:00:28.232683476Z - honor_labels: true
2024-06-24T18:00:28.232686404Z   job_name: pushgateway
2024-06-24T18:00:28.232689211Z   scrape_interval: 30s
2024-06-24T18:00:28.232692178Z   static_configs:
2024-06-24T18:00:28.232694970Z   - targets:
2024-06-24T18:00:28.232698643Z     - pushgateway-prometheus-pushgateway:9091
2024-06-24T18:00:28.232701612Z - job_name: prometheus
2024-06-24T18:00:28.232704412Z   static_configs:
2024-06-24T18:00:28.232707208Z   - targets:
2024-06-24T18:00:28.232710103Z     - localhost:9090
2024-06-24T18:00:28.232712907Z - job_name: node
2024-06-24T18:00:28.232715798Z   static_configs:
2024-06-24T18:00:28.232719110Z   - targets:
2024-06-24T18:00:28.232722149Z     - prometheus-node-exporter:9100
2024-06-24T18:00:28.232729473Z 
2024-06-24T18:00:28.238465016Z Alerts read: 2
2024-06-24T18:00:28.240700000Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-06-24T18:00:28.244237242Z groups:
2024-06-24T18:00:28.244250067Z - name: osm_alert_rules
2024-06-24T18:00:28.244253375Z   rules:
2024-06-24T18:00:28.244256397Z   - alert: vdu_down
2024-06-24T18:00:28.244259326Z     annotations:
2024-06-24T18:00:28.244263368Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-06-24T18:00:28.244268731Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-06-24T18:00:28.244271712Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-06-24T18:00:28.244282610Z     expr: vm_status_extended != 1
2024-06-24T18:00:28.244286081Z     for: 3m
2024-06-24T18:00:28.244289109Z - name: _osm_alert_rules_20240624180028_
2024-06-24T18:00:28.244292087Z   rules:
2024-06-24T18:00:28.244295650Z   - alert: scalein_78e8af7a-85b9-4769-a672-4d18904ac3e1
2024-06-24T18:00:28.244298814Z     expr: (count (osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:28.244305292Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:28.244308723Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-06-24T18:00:28.244311694Z     for: 1m
2024-06-24T18:00:28.244314534Z     labels:
2024-06-24T18:00:28.244317486Z       ns_id: 02dc7f3e-c632-4b27-b418-965aa0c8ca0f
2024-06-24T18:00:28.244320499Z       vdu_id: hackfest_basic_metrics-VM
2024-06-24T18:00:28.244335447Z       vnf_member_index: vnf
2024-06-24T18:00:28.244338495Z   - alert: scaleout_9e672951-fc6a-4a32-a53b-f1c996b6eead
2024-06-24T18:00:28.244341480Z     expr: (count (osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:28.244344519Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="02dc7f3e-c632-4b27-b418-965aa0c8ca0f",
2024-06-24T18:00:28.244347806Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-06-24T18:00:28.244350729Z     for: 1m
2024-06-24T18:00:28.244353531Z     labels:
2024-06-24T18:00:28.244356483Z       ns_id: 02dc7f3e-c632-4b27-b418-965aa0c8ca0f
2024-06-24T18:00:28.244359249Z       vdu_id: hackfest_basic_metrics-VM
2024-06-24T18:00:28.244362102Z       vnf_member_index: vnf
2024-06-24T18:00:28.244369622Z 
2024-06-24T18:00:28.247924934Z New config written, updating prometheus
2024-06-24T18:00:33.292646192Z Prometheus config update successful
2024-06-24T18:00:33.292663987Z 
2024-06-24T18:15:54.643617135Z Changes detected, updating prometheus config
2024-06-24T18:15:54.646775821Z Jobs detected: 0
2024-06-24T18:15:54.652398952Z Writing new config file to /etc/prometheus/prometheus.yml
2024-06-24T18:15:54.654896830Z alerting:
2024-06-24T18:15:54.654910595Z   alertmanagers:
2024-06-24T18:15:54.654915997Z   - static_configs:
2024-06-24T18:15:54.654920253Z     - targets:
2024-06-24T18:15:54.654925281Z       - alertmanager:9093
2024-06-24T18:15:54.654929575Z global:
2024-06-24T18:15:54.654933855Z   evaluation_interval: 15s
2024-06-24T18:15:54.654938147Z   scrape_interval: 15s
2024-06-24T18:15:54.654942456Z rule_files:
2024-06-24T18:15:54.654946667Z - osm_metric_rules.yml
2024-06-24T18:15:54.654950959Z - osm_alert_rules.yml
2024-06-24T18:15:54.654955555Z scrape_configs:
2024-06-24T18:15:54.654959770Z - job_name: mon_exporter
2024-06-24T18:15:54.654964073Z   static_configs:
2024-06-24T18:15:54.654968328Z   - targets:
2024-06-24T18:15:54.654972509Z     - mon:8000
2024-06-24T18:15:54.654976725Z - honor_labels: true
2024-06-24T18:15:54.654980983Z   job_name: pushgateway
2024-06-24T18:15:54.654985158Z   scrape_interval: 30s
2024-06-24T18:15:54.654989329Z   static_configs:
2024-06-24T18:15:54.654993466Z   - targets:
2024-06-24T18:15:54.654998931Z     - pushgateway-prometheus-pushgateway:9091
2024-06-24T18:15:54.655003208Z - job_name: prometheus
2024-06-24T18:15:54.655007364Z   static_configs:
2024-06-24T18:15:54.655011583Z   - targets:
2024-06-24T18:15:54.655015780Z     - localhost:9090
2024-06-24T18:15:54.655019994Z - job_name: node
2024-06-24T18:15:54.655024141Z   static_configs:
2024-06-24T18:15:54.655028274Z   - targets:
2024-06-24T18:15:54.655032520Z     - prometheus-node-exporter:9100
2024-06-24T18:15:54.655036492Z 
2024-06-24T18:15:54.658569194Z Alerts read: 0
2024-06-24T18:15:54.660310396Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-06-24T18:15:54.661392175Z groups:
2024-06-24T18:15:54.661401228Z - name: osm_alert_rules
2024-06-24T18:15:54.661405019Z   rules:
2024-06-24T18:15:54.661408771Z   - alert: vdu_down
2024-06-24T18:15:54.661412249Z     annotations:
2024-06-24T18:15:54.661416673Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-06-24T18:15:54.661421217Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-06-24T18:15:54.661424753Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-06-24T18:15:54.661442563Z     expr: vm_status_extended != 1
2024-06-24T18:15:54.661446256Z     for: 3m
2024-06-24T18:15:54.661449495Z 
2024-06-24T18:15:54.662310431Z New config written, updating prometheus
2024-06-24T18:15:59.697680484Z Prometheus config update successful
2024-06-24T18:15:59.697712764Z 
2024-06-24T18:15:59.697716419Z Changes detected, updating prometheus config
2024-06-24T18:15:59.702161050Z Jobs detected: 0
2024-06-24T18:15:59.706552999Z Writing new config file to /etc/prometheus/prometheus.yml
2024-06-24T18:15:59.708104705Z alerting:
2024-06-24T18:15:59.708321422Z   alertmanagers:
2024-06-24T18:15:59.708326377Z   - static_configs:
2024-06-24T18:15:59.708329385Z     - targets:
2024-06-24T18:15:59.708333564Z       - alertmanager:9093
2024-06-24T18:15:59.708336551Z global:
2024-06-24T18:15:59.708339479Z   evaluation_interval: 15s
2024-06-24T18:15:59.709588079Z   scrape_interval: 15s
2024-06-24T18:15:59.709598513Z rule_files:
2024-06-24T18:15:59.709601689Z - osm_metric_rules.yml
2024-06-24T18:15:59.709604738Z - osm_alert_rules.yml
2024-06-24T18:15:59.709607671Z scrape_configs:
2024-06-24T18:15:59.709610640Z - job_name: mon_exporter
2024-06-24T18:15:59.709613951Z   static_configs:
2024-06-24T18:15:59.709617098Z   - targets:
2024-06-24T18:15:59.709620163Z     - mon:8000
2024-06-24T18:15:59.709623259Z - honor_labels: true
2024-06-24T18:15:59.709676729Z   job_name: pushgateway
2024-06-24T18:15:59.709679691Z   scrape_interval: 30s
2024-06-24T18:15:59.709682429Z   static_configs:
2024-06-24T18:15:59.709685243Z   - targets:
2024-06-24T18:15:59.709690632Z     - pushgateway-prometheus-pushgateway:9091
2024-06-24T18:15:59.709693508Z - job_name: prometheus
2024-06-24T18:15:59.709696393Z   static_configs:
2024-06-24T18:15:59.709699102Z   - targets:
2024-06-24T18:15:59.709701746Z     - localhost:9090
2024-06-24T18:15:59.709704478Z - job_name: node
2024-06-24T18:15:59.709707186Z   static_configs:
2024-06-24T18:15:59.709709957Z   - targets:
2024-06-24T18:15:59.709712721Z     - prometheus-node-exporter:9100
2024-06-24T18:15:59.709715333Z 
2024-06-24T18:15:59.720105368Z Alerts read: 0
2024-06-24T18:15:59.724216366Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-06-24T18:15:59.728157507Z groups:
2024-06-24T18:15:59.728170941Z - name: osm_alert_rules
2024-06-24T18:15:59.728174493Z   rules:
2024-06-24T18:15:59.728178141Z   - alert: vdu_down
2024-06-24T18:15:59.728181798Z     annotations:
2024-06-24T18:15:59.728185526Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-06-24T18:15:59.728189393Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-06-24T18:15:59.728192808Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-06-24T18:15:59.728196578Z     expr: vm_status_extended != 1
2024-06-24T18:15:59.728199994Z     for: 3m
2024-06-24T18:15:59.728203156Z 
2024-06-24T18:15:59.728206598Z New config written, updating prometheus
2024-06-24T18:16:04.757361504Z Prometheus config update successful
2024-06-24T18:16:04.757391499Z