2024-05-24T09:43:11.033507465Z level=info ts=2024-05-24T09:43:11.03332431Z caller=main.go:137 msg="Starting prometheus-config-reloader" version="(version=0.71.2, branch=refs/tags/v0.71.2, revision=af2014407bdc25c4fc2b26cd99c9655235ebdf88)"
2024-05-24T09:43:11.033539996Z level=info ts=2024-05-24T09:43:11.033388268Z caller=main.go:138 build_context="(go=go1.21.6, platform=linux/amd64, user=Action-Run-ID-7656327832, date=20240125-14:58:54, tags=unknown)"
2024-05-24T09:43:11.037705835Z level=info ts=2024-05-24T09:43:11.037525262Z caller=reloader.go:246 msg="reloading via HTTP"
2024-05-24T09:43:11.037864725Z level=info ts=2024-05-24T09:43:11.037775115Z caller=reloader.go:282 msg="started watching config file and directories for changes" cfg= out= dirs=/etc/config
2024-05-24T09:46:11.049144823Z level=info ts=2024-05-24T09:46:11.048911364Z caller=reloader.go:424 msg="Reload triggered" cfg_in= cfg_out= watched_dirs=/etc/config
2024-05-24T09:44:10.577574596Z ts=2024-05-24T09:44:10.577Z caller=main.go:588 level=info msg="Starting Prometheus Server" mode=server version="(version=2.49.1, branch=HEAD, revision=43e14844a33b65e2a396e3944272af8b3a494071)"
2024-05-24T09:44:10.577788670Z ts=2024-05-24T09:44:10.577Z caller=main.go:593 level=info build_context="(go=go1.21.6, platform=linux/amd64, user=root@6d5f4c649d25, date=20240115-16:58:43, tags=netgo,builtinassets,stringlabels)"
2024-05-24T09:44:10.578072576Z ts=2024-05-24T09:44:10.577Z caller=main.go:594 level=info host_details="(Linux 5.15.0-47-generic #51-Ubuntu SMP Thu Aug 11 07:51:15 UTC 2022 x86_64 prometheus-0 (none))"
2024-05-24T09:44:10.578175898Z ts=2024-05-24T09:44:10.578Z caller=main.go:595 level=info fd_limits="(soft=1048576, hard=1048576)"
2024-05-24T09:44:10.578278277Z ts=2024-05-24T09:44:10.578Z caller=main.go:596 level=info vm_limits="(soft=unlimited, hard=unlimited)"
2024-05-24T09:44:10.580750154Z ts=2024-05-24T09:44:10.580Z caller=web.go:565 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-05-24T09:44:10.583481943Z ts=2024-05-24T09:44:10.583Z caller=main.go:1039 level=info msg="Starting TSDB ..."
2024-05-24T09:44:10.594100815Z ts=2024-05-24T09:44:10.593Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9090
2024-05-24T09:44:10.594115544Z ts=2024-05-24T09:44:10.594Z caller=tls_config.go:277 level=info component=web msg="TLS is disabled." http2=false address=[::]:9090
2024-05-24T09:44:10.594150789Z ts=2024-05-24T09:44:10.594Z caller=head.go:606 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-05-24T09:44:10.594154774Z ts=2024-05-24T09:44:10.594Z caller=head.go:687 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.687µs
2024-05-24T09:44:10.594197052Z ts=2024-05-24T09:44:10.594Z caller=head.go:695 level=info component=tsdb msg="Replaying WAL, this may take a while"
2024-05-24T09:44:10.595356840Z ts=2024-05-24T09:44:10.595Z caller=head.go:766 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-05-24T09:44:10.595370129Z ts=2024-05-24T09:44:10.595Z caller=head.go:803 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.301µs wal_replay_duration=1.106949ms wbl_replay_duration=159ns total_replay_duration=1.180781ms
2024-05-24T09:44:10.599883436Z ts=2024-05-24T09:44:10.599Z caller=main.go:1060 level=info fs_type=EXT4_SUPER_MAGIC
2024-05-24T09:44:10.599897820Z ts=2024-05-24T09:44:10.599Z caller=main.go:1063 level=info msg="TSDB started"
2024-05-24T09:44:10.599901379Z ts=2024-05-24T09:44:10.599Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T09:44:10.606313040Z ts=2024-05-24T09:44:10.606Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=6.3641ms db_storage=1.413µs remote_storage=2.037µs web_handler=512ns query_engine=1.6µs scrape=3.343375ms scrape_sd=126.335µs notify=37.213µs notify_sd=10.007µs rules=2.346841ms tracing=13.529µs
2024-05-24T09:44:10.606734751Z ts=2024-05-24T09:44:10.606Z caller=main.go:1024 level=info msg="Server is ready to receive web requests."
2024-05-24T09:44:10.606750251Z ts=2024-05-24T09:44:10.606Z caller=manager.go:146 level=info component="rule manager" msg="Starting rule manager..."
2024-05-24T09:45:47.931137876Z ts=2024-05-24T09:45:47.930Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T09:45:47.937269294Z ts=2024-05-24T09:45:47.935Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.842626ms db_storage=8.138µs remote_storage=2.085µs web_handler=634ns query_engine=1.551µs scrape=253.755µs scrape_sd=114.805µs notify=74.494µs notify_sd=7.69µs rules=2.592262ms tracing=18.838µs
2024-05-24T09:46:11.046513272Z ts=2024-05-24T09:46:11.046Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T09:46:11.048459988Z ts=2024-05-24T09:46:11.048Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.026054ms db_storage=2.282µs remote_storage=2.163µs web_handler=842ns query_engine=2.486µs scrape=225.183µs scrape_sd=79.148µs notify=32.81µs notify_sd=15.344µs rules=1.181048ms tracing=7.671µs
2024-05-24T09:59:22.382428532Z ts=2024-05-24T09:59:22.382Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T09:59:22.386634251Z ts=2024-05-24T09:59:22.386Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.264983ms db_storage=6.467µs remote_storage=1.92µs web_handler=656ns query_engine=1.441µs scrape=487.533µs scrape_sd=198.453µs notify=102.743µs notify_sd=9.783µs rules=2.56298ms tracing=19.535µs
2024-05-24T09:59:27.435604500Z ts=2024-05-24T09:59:27.435Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T09:59:27.437531243Z ts=2024-05-24T09:59:27.437Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.945535ms db_storage=1.903µs remote_storage=1.779µs web_handler=453ns query_engine=1.473µs scrape=103.82µs scrape_sd=47.47µs notify=41.19µs notify_sd=30.706µs rules=1.38758ms tracing=8.263µs
2024-05-24T10:12:17.583162845Z ts=2024-05-24T10:12:17.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:12:32.582606760Z ts=2024-05-24T10:12:32.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:12:47.582509654Z ts=2024-05-24T10:12:47.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:13:02.582821281Z ts=2024-05-24T10:13:02.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:13:17.584105529Z ts=2024-05-24T10:13:17.583Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:13:32.582801386Z ts=2024-05-24T10:13:32.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:13:47.582473065Z ts=2024-05-24T10:13:47.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:14:02.582792418Z ts=2024-05-24T10:14:02.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:14:17.582936419Z ts=2024-05-24T10:14:17.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:14:32.582863616Z ts=2024-05-24T10:14:32.582Z caller=group.go:462 level=warn name=vm_status_extended index=0 component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"f1560959-9748-41e9-870e-b63fee2c5d7c\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0d823150-94eb-4194-97d0-8741a6b8d82e\", project_id=\"b12e4086-f5f4-4e6a-af47-cb63f24bc741\", vdu_id=\"bcd9e2b6-3a4b-4cdb-b44b-654eb47b90d7\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"14ed06f0-9208-4724-a753-d1254f1de329\", vm_id=\"f16cfbcf-9829-4cbd-bd73-b023f5299583\", vnf_id=\"1184b3ca-c33c-4838-a7c7-79a6e593de8a\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-05-24T10:15:55.233599529Z ts=2024-05-24T10:15:55.233Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T10:15:55.239725129Z ts=2024-05-24T10:15:55.239Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=6.146722ms db_storage=11.133µs remote_storage=2.628µs web_handler=1.218µs query_engine=2.09µs scrape=509.403µs scrape_sd=177.697µs notify=428.143µs notify_sd=41.923µs rules=2.888936ms tracing=76.791µs
2024-05-24T10:16:00.295380009Z ts=2024-05-24T10:16:00.295Z caller=main.go:1245 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-05-24T10:16:00.296985228Z ts=2024-05-24T10:16:00.296Z caller=main.go:1282 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.612326ms db_storage=1.601µs remote_storage=2.354µs web_handler=456ns query_engine=1.487µs scrape=108.791µs scrape_sd=50.399µs notify=31.299µs notify_sd=7.237µs rules=1.043875ms tracing=6.001µs
2024-05-24T09:45:47.860937209Z Created MongoClient to connect to MongoDB!
2024-05-24T09:45:47.860970493Z Generating prometheus config files
2024-05-24T09:45:47.901079914Z Jobs detected: 0
2024-05-24T09:45:47.907261298Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-24T09:45:47.915695764Z alerting:
2024-05-24T09:45:47.915709006Z   alertmanagers:
2024-05-24T09:45:47.915712832Z   - static_configs:
2024-05-24T09:45:47.915715881Z     - targets:
2024-05-24T09:45:47.915719404Z       - alertmanager:9093
2024-05-24T09:45:47.915722477Z global:
2024-05-24T09:45:47.915725485Z   evaluation_interval: 15s
2024-05-24T09:45:47.915728510Z   scrape_interval: 15s
2024-05-24T09:45:47.915731953Z rule_files:
2024-05-24T09:45:47.915734962Z - osm_metric_rules.yml
2024-05-24T09:45:47.915737910Z - osm_alert_rules.yml
2024-05-24T09:45:47.915740919Z scrape_configs:
2024-05-24T09:45:47.915744294Z - job_name: mon_exporter
2024-05-24T09:45:47.915747262Z   static_configs:
2024-05-24T09:45:47.915750250Z   - targets:
2024-05-24T09:45:47.915753312Z     - mon:8000
2024-05-24T09:45:47.915756321Z - honor_labels: true
2024-05-24T09:45:47.915759290Z   job_name: pushgateway
2024-05-24T09:45:47.915762201Z   scrape_interval: 30s
2024-05-24T09:45:47.915765092Z   static_configs:
2024-05-24T09:45:47.915768073Z   - targets:
2024-05-24T09:45:47.915771709Z     - pushgateway-prometheus-pushgateway:9091
2024-05-24T09:45:47.915774656Z - job_name: prometheus
2024-05-24T09:45:47.915777686Z   static_configs:
2024-05-24T09:45:47.915780619Z   - targets:
2024-05-24T09:45:47.915784033Z     - localhost:9090
2024-05-24T09:45:47.915787209Z - job_name: node
2024-05-24T09:45:47.915790120Z   static_configs:
2024-05-24T09:45:47.915793040Z   - targets:
2024-05-24T09:45:47.915796095Z     - prometheus-node-exporter:9100
2024-05-24T09:45:47.915798934Z 
2024-05-24T09:45:47.921596616Z Alerts read: 0
2024-05-24T09:45:47.923095687Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-24T09:45:47.924247794Z groups:
2024-05-24T09:45:47.924257869Z - name: osm_alert_rules
2024-05-24T09:45:47.924261198Z   rules:
2024-05-24T09:45:47.924264328Z   - alert: vdu_down
2024-05-24T09:45:47.924267481Z     annotations:
2024-05-24T09:45:47.924271460Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-24T09:45:47.924277590Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-24T09:45:47.924280757Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-24T09:45:47.924284476Z     expr: vm_status_extended != 1
2024-05-24T09:45:47.924287952Z     for: 3m
2024-05-24T09:45:47.924290819Z 
2024-05-24T09:45:47.925559333Z New config written, updating prometheus
2024-05-24T09:45:52.966282529Z Prometheus config update successful
2024-05-24T09:45:57.981852492Z Listening to changes in prometheus jobs and alerts collections
2024-05-24T09:59:22.346527168Z Changes detected, updating prometheus config
2024-05-24T09:59:22.349932445Z Jobs detected: 0
2024-05-24T09:59:22.355334941Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-24T09:59:22.359075065Z alerting:
2024-05-24T09:59:22.359088678Z   alertmanagers:
2024-05-24T09:59:22.359093214Z   - static_configs:
2024-05-24T09:59:22.359096851Z     - targets:
2024-05-24T09:59:22.359100995Z       - alertmanager:9093
2024-05-24T09:59:22.359104701Z global:
2024-05-24T09:59:22.359111845Z   evaluation_interval: 15s
2024-05-24T09:59:22.359115493Z   scrape_interval: 15s
2024-05-24T09:59:22.359119084Z rule_files:
2024-05-24T09:59:22.359135083Z - osm_metric_rules.yml
2024-05-24T09:59:22.359138712Z - osm_alert_rules.yml
2024-05-24T09:59:22.359142297Z scrape_configs:
2024-05-24T09:59:22.359145845Z - job_name: mon_exporter
2024-05-24T09:59:22.359149379Z   static_configs:
2024-05-24T09:59:22.359152843Z   - targets:
2024-05-24T09:59:22.359156743Z     - mon:8000
2024-05-24T09:59:22.359160205Z - honor_labels: true
2024-05-24T09:59:22.359163872Z   job_name: pushgateway
2024-05-24T09:59:22.359167627Z   scrape_interval: 30s
2024-05-24T09:59:22.359171140Z   static_configs:
2024-05-24T09:59:22.359174661Z   - targets:
2024-05-24T09:59:22.359178863Z     - pushgateway-prometheus-pushgateway:9091
2024-05-24T09:59:22.359182277Z - job_name: prometheus
2024-05-24T09:59:22.359185678Z   static_configs:
2024-05-24T09:59:22.359189133Z   - targets:
2024-05-24T09:59:22.359192582Z     - localhost:9090
2024-05-24T09:59:22.359195977Z - job_name: node
2024-05-24T09:59:22.359199431Z   static_configs:
2024-05-24T09:59:22.359202840Z   - targets:
2024-05-24T09:59:22.359206381Z     - prometheus-node-exporter:9100
2024-05-24T09:59:22.359209672Z 
2024-05-24T09:59:22.368382173Z Alerts read: 2
2024-05-24T09:59:22.370996371Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-24T09:59:22.374547044Z groups:
2024-05-24T09:59:22.374557766Z - name: osm_alert_rules
2024-05-24T09:59:22.374560939Z   rules:
2024-05-24T09:59:22.374564139Z   - alert: vdu_down
2024-05-24T09:59:22.374567096Z     annotations:
2024-05-24T09:59:22.374571019Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-24T09:59:22.374574784Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-24T09:59:22.374577675Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-24T09:59:22.374581105Z     expr: vm_status_extended != 1
2024-05-24T09:59:22.374584044Z     for: 3m
2024-05-24T09:59:22.374587046Z - name: _osm_alert_rules_20240524095922_
2024-05-24T09:59:22.374590020Z   rules:
2024-05-24T09:59:22.374593452Z   - alert: scalein_25b3e8c7-60fd-4651-af5c-9614be3f36ec
2024-05-24T09:59:22.374596329Z     expr: (count (osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:22.374599937Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:22.374604153Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-05-24T09:59:22.374607024Z     for: 1m
2024-05-24T09:59:22.374609914Z     labels:
2024-05-24T09:59:22.374613114Z       ns_id: 0d823150-94eb-4194-97d0-8741a6b8d82e
2024-05-24T09:59:22.374616230Z       vdu_id: hackfest_basic_metrics-VM
2024-05-24T09:59:22.374622311Z       vnf_member_index: vnf
2024-05-24T09:59:22.374625245Z   - alert: scaleout_3593d883-22ae-4ed1-8a33-e546138c5c8e
2024-05-24T09:59:22.374628299Z     expr: (count (osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:22.374631364Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:22.374634518Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-05-24T09:59:22.374637694Z     for: 1m
2024-05-24T09:59:22.374640557Z     labels:
2024-05-24T09:59:22.374643559Z       ns_id: 0d823150-94eb-4194-97d0-8741a6b8d82e
2024-05-24T09:59:22.374646430Z       vdu_id: hackfest_basic_metrics-VM
2024-05-24T09:59:22.374649279Z       vnf_member_index: vnf
2024-05-24T09:59:22.374665881Z 
2024-05-24T09:59:22.377802449Z New config written, updating prometheus
2024-05-24T09:59:27.409185306Z Prometheus config update successful
2024-05-24T09:59:27.409212466Z 
2024-05-24T09:59:27.411659248Z Changes detected, updating prometheus config
2024-05-24T09:59:27.412698242Z Jobs detected: 0
2024-05-24T09:59:27.417182684Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-24T09:59:27.419454248Z alerting:
2024-05-24T09:59:27.419479020Z   alertmanagers:
2024-05-24T09:59:27.419483184Z   - static_configs:
2024-05-24T09:59:27.419486313Z     - targets:
2024-05-24T09:59:27.419489887Z       - alertmanager:9093
2024-05-24T09:59:27.419493030Z global:
2024-05-24T09:59:27.419496313Z   evaluation_interval: 15s
2024-05-24T09:59:27.419499321Z   scrape_interval: 15s
2024-05-24T09:59:27.419502477Z rule_files:
2024-05-24T09:59:27.419505431Z - osm_metric_rules.yml
2024-05-24T09:59:27.419508811Z - osm_alert_rules.yml
2024-05-24T09:59:27.419512096Z scrape_configs:
2024-05-24T09:59:27.419515021Z - job_name: mon_exporter
2024-05-24T09:59:27.419517990Z   static_configs:
2024-05-24T09:59:27.419521070Z   - targets:
2024-05-24T09:59:27.419524022Z     - mon:8000
2024-05-24T09:59:27.419527046Z - honor_labels: true
2024-05-24T09:59:27.419529986Z   job_name: pushgateway
2024-05-24T09:59:27.419532855Z   scrape_interval: 30s
2024-05-24T09:59:27.419536370Z   static_configs:
2024-05-24T09:59:27.419539348Z   - targets:
2024-05-24T09:59:27.419543026Z     - pushgateway-prometheus-pushgateway:9091
2024-05-24T09:59:27.419546036Z - job_name: prometheus
2024-05-24T09:59:27.419549001Z   static_configs:
2024-05-24T09:59:27.419551972Z   - targets:
2024-05-24T09:59:27.419554911Z     - localhost:9090
2024-05-24T09:59:27.419557889Z - job_name: node
2024-05-24T09:59:27.419560851Z   static_configs:
2024-05-24T09:59:27.419563765Z   - targets:
2024-05-24T09:59:27.419566722Z     - prometheus-node-exporter:9100
2024-05-24T09:59:27.419580224Z 
2024-05-24T09:59:27.424295964Z Alerts read: 2
2024-05-24T09:59:27.426105625Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-24T09:59:27.429113893Z groups:
2024-05-24T09:59:27.429130231Z - name: osm_alert_rules
2024-05-24T09:59:27.429135122Z   rules:
2024-05-24T09:59:27.429139378Z   - alert: vdu_down
2024-05-24T09:59:27.429143562Z     annotations:
2024-05-24T09:59:27.429148593Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-24T09:59:27.429153786Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-24T09:59:27.429158142Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-24T09:59:27.429162991Z     expr: vm_status_extended != 1
2024-05-24T09:59:27.429167046Z     for: 3m
2024-05-24T09:59:27.429171169Z - name: _osm_alert_rules_20240524095927_
2024-05-24T09:59:27.429175226Z   rules:
2024-05-24T09:59:27.429179814Z   - alert: scalein_25b3e8c7-60fd-4651-af5c-9614be3f36ec
2024-05-24T09:59:27.429183859Z     expr: (count (osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:27.429188616Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:27.429192641Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-05-24T09:59:27.429196923Z     for: 1m
2024-05-24T09:59:27.429201217Z     labels:
2024-05-24T09:59:27.429205214Z       ns_id: 0d823150-94eb-4194-97d0-8741a6b8d82e
2024-05-24T09:59:27.429223388Z       vdu_id: hackfest_basic_metrics-VM
2024-05-24T09:59:27.429228034Z       vnf_member_index: vnf
2024-05-24T09:59:27.429232311Z   - alert: scaleout_3593d883-22ae-4ed1-8a33-e546138c5c8e
2024-05-24T09:59:27.429236336Z     expr: (count (osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:27.429240392Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="0d823150-94eb-4194-97d0-8741a6b8d82e",
2024-05-24T09:59:27.429244413Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-05-24T09:59:27.429248418Z     for: 1m
2024-05-24T09:59:27.429252511Z     labels:
2024-05-24T09:59:27.429256575Z       ns_id: 0d823150-94eb-4194-97d0-8741a6b8d82e
2024-05-24T09:59:27.429261082Z       vdu_id: hackfest_basic_metrics-VM
2024-05-24T09:59:27.429265080Z       vnf_member_index: vnf
2024-05-24T09:59:27.429268849Z 
2024-05-24T09:59:27.432397716Z New config written, updating prometheus
2024-05-24T09:59:32.472579993Z Prometheus config update successful
2024-05-24T09:59:32.472596797Z 
2024-05-24T10:15:55.206963816Z Changes detected, updating prometheus config
2024-05-24T10:15:55.210647505Z Jobs detected: 0
2024-05-24T10:15:55.216646717Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-24T10:15:55.219675118Z alerting:
2024-05-24T10:15:55.219689187Z   alertmanagers:
2024-05-24T10:15:55.219693329Z   - static_configs:
2024-05-24T10:15:55.219724303Z     - targets:
2024-05-24T10:15:55.219733461Z       - alertmanager:9093
2024-05-24T10:15:55.219738763Z global:
2024-05-24T10:15:55.219744450Z   evaluation_interval: 15s
2024-05-24T10:15:55.219750096Z   scrape_interval: 15s
2024-05-24T10:15:55.219755566Z rule_files:
2024-05-24T10:15:55.219759460Z - osm_metric_rules.yml
2024-05-24T10:15:55.219762800Z - osm_alert_rules.yml
2024-05-24T10:15:55.219766075Z scrape_configs:
2024-05-24T10:15:55.219769436Z - job_name: mon_exporter
2024-05-24T10:15:55.219772617Z   static_configs:
2024-05-24T10:15:55.219776078Z   - targets:
2024-05-24T10:15:55.219779458Z     - mon:8000
2024-05-24T10:15:55.219782590Z - honor_labels: true
2024-05-24T10:15:55.219785910Z   job_name: pushgateway
2024-05-24T10:15:55.219789134Z   scrape_interval: 30s
2024-05-24T10:15:55.219792354Z   static_configs:
2024-05-24T10:15:55.219795650Z   - targets:
2024-05-24T10:15:55.219799611Z     - pushgateway-prometheus-pushgateway:9091
2024-05-24T10:15:55.219802842Z - job_name: prometheus
2024-05-24T10:15:55.219806124Z   static_configs:
2024-05-24T10:15:55.219809317Z   - targets:
2024-05-24T10:15:55.219816845Z     - localhost:9090
2024-05-24T10:15:55.219824040Z - job_name: node
2024-05-24T10:15:55.219829068Z   static_configs:
2024-05-24T10:15:55.219834087Z   - targets:
2024-05-24T10:15:55.219839111Z     - prometheus-node-exporter:9100
2024-05-24T10:15:55.219843568Z 
2024-05-24T10:15:55.223700066Z Alerts read: 0
2024-05-24T10:15:55.225768422Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-24T10:15:55.226793358Z groups:
2024-05-24T10:15:55.226805835Z - name: osm_alert_rules
2024-05-24T10:15:55.226809604Z   rules:
2024-05-24T10:15:55.226812852Z   - alert: vdu_down
2024-05-24T10:15:55.226816002Z     annotations:
2024-05-24T10:15:55.226857661Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-24T10:15:55.226862278Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-24T10:15:55.226865560Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-24T10:15:55.226884668Z     expr: vm_status_extended != 1
2024-05-24T10:15:55.226887986Z     for: 3m
2024-05-24T10:15:55.226891209Z 
2024-05-24T10:15:55.227806333Z New config written, updating prometheus
2024-05-24T10:16:00.275668075Z Prometheus config update successful
2024-05-24T10:16:00.275684605Z 
2024-05-24T10:16:00.275687989Z Changes detected, updating prometheus config
2024-05-24T10:16:00.277013823Z Jobs detected: 0
2024-05-24T10:16:00.281565004Z Writing new config file to /etc/prometheus/prometheus.yml
2024-05-24T10:16:00.284158557Z alerting:
2024-05-24T10:16:00.284171050Z   alertmanagers:
2024-05-24T10:16:00.284174740Z   - static_configs:
2024-05-24T10:16:00.284178533Z     - targets:
2024-05-24T10:16:00.284182694Z       - alertmanager:9093
2024-05-24T10:16:00.284185746Z global:
2024-05-24T10:16:00.284188813Z   evaluation_interval: 15s
2024-05-24T10:16:00.284191743Z   scrape_interval: 15s
2024-05-24T10:16:00.284194703Z rule_files:
2024-05-24T10:16:00.284197858Z - osm_metric_rules.yml
2024-05-24T10:16:00.284200833Z - osm_alert_rules.yml
2024-05-24T10:16:00.284203839Z scrape_configs:
2024-05-24T10:16:00.284206947Z - job_name: mon_exporter
2024-05-24T10:16:00.284209867Z   static_configs:
2024-05-24T10:16:00.284227240Z   - targets:
2024-05-24T10:16:00.284231949Z     - mon:8000
2024-05-24T10:16:00.284236331Z - honor_labels: true
2024-05-24T10:16:00.284239873Z   job_name: pushgateway
2024-05-24T10:16:00.284242857Z   scrape_interval: 30s
2024-05-24T10:16:00.284245826Z   static_configs:
2024-05-24T10:16:00.284248771Z   - targets:
2024-05-24T10:16:00.284252359Z     - pushgateway-prometheus-pushgateway:9091
2024-05-24T10:16:00.284255407Z - job_name: prometheus
2024-05-24T10:16:00.284262786Z   static_configs:
2024-05-24T10:16:00.284265980Z   - targets:
2024-05-24T10:16:00.284269014Z     - localhost:9090
2024-05-24T10:16:00.284271950Z - job_name: node
2024-05-24T10:16:00.284275043Z   static_configs:
2024-05-24T10:16:00.284278039Z   - targets:
2024-05-24T10:16:00.284281363Z     - prometheus-node-exporter:9100
2024-05-24T10:16:00.284284273Z 
2024-05-24T10:16:00.288236966Z Alerts read: 0
2024-05-24T10:16:00.290241548Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-05-24T10:16:00.291472534Z groups:
2024-05-24T10:16:00.291481880Z - name: osm_alert_rules
2024-05-24T10:16:00.291485296Z   rules:
2024-05-24T10:16:00.291488449Z   - alert: vdu_down
2024-05-24T10:16:00.291491479Z     annotations:
2024-05-24T10:16:00.291495243Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-05-24T10:16:00.291498899Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-05-24T10:16:00.291502152Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-05-24T10:16:00.291506100Z     expr: vm_status_extended != 1
2024-05-24T10:16:00.291509115Z     for: 3m
2024-05-24T10:16:00.291511980Z 
2024-05-24T10:16:00.292605074Z New config written, updating prometheus
2024-05-24T10:16:05.325375825Z Prometheus config update successful
2024-05-24T10:16:05.325486363Z