2024-04-03T01:25:24.649063899Z ts=2024-04-03T01:25:24.648Z caller=main.go:539 level=info msg="No time or size retention was set so using the default time retention" duration=15d
2024-04-03T01:25:24.649096787Z ts=2024-04-03T01:25:24.648Z caller=main.go:583 level=info msg="Starting Prometheus Server" mode=server version="(version=2.47.0, branch=HEAD, revision=efa34a5840661c29c2e362efa76bc3a70dccb335)"
2024-04-03T01:25:24.649212669Z ts=2024-04-03T01:25:24.648Z caller=main.go:588 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=root@409eb5e6b30c, date=20230906-10:20:07, tags=netgo,builtinassets,stringlabels)"
2024-04-03T01:25:24.649293977Z ts=2024-04-03T01:25:24.649Z caller=main.go:589 level=info host_details="(Linux 5.15.0-47-generic #51-Ubuntu SMP Thu Aug 11 07:51:15 UTC 2022 x86_64 prometheus-0 (none))"
2024-04-03T01:25:24.649369834Z ts=2024-04-03T01:25:24.649Z caller=main.go:590 level=info fd_limits="(soft=1048576, hard=1048576)"
2024-04-03T01:25:24.649449276Z ts=2024-04-03T01:25:24.649Z caller=main.go:591 level=info vm_limits="(soft=unlimited, hard=unlimited)"
2024-04-03T01:25:24.653013377Z ts=2024-04-03T01:25:24.652Z caller=web.go:566 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
2024-04-03T01:25:24.654023376Z ts=2024-04-03T01:25:24.653Z caller=main.go:1024 level=info msg="Starting TSDB ..."
2024-04-03T01:25:24.655385144Z ts=2024-04-03T01:25:24.655Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9090
2024-04-03T01:25:24.655394447Z ts=2024-04-03T01:25:24.655Z caller=tls_config.go:277 level=info component=web msg="TLS is disabled." http2=false address=[::]:9090
2024-04-03T01:25:24.661629606Z ts=2024-04-03T01:25:24.661Z caller=head.go:600 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2024-04-03T01:25:24.661643863Z ts=2024-04-03T01:25:24.661Z caller=head.go:681 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.354µs
2024-04-03T01:25:24.661647150Z ts=2024-04-03T01:25:24.661Z caller=head.go:689 level=info component=tsdb msg="Replaying WAL, this may take a while"
2024-04-03T01:25:24.662204435Z ts=2024-04-03T01:25:24.662Z caller=head.go:760 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2024-04-03T01:25:24.662215396Z ts=2024-04-03T01:25:24.662Z caller=head.go:797 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=53.102µs wal_replay_duration=492.6µs wbl_replay_duration=202ns total_replay_duration=573.586µs
2024-04-03T01:25:24.672125486Z ts=2024-04-03T01:25:24.672Z caller=main.go:1045 level=info fs_type=EXT4_SUPER_MAGIC
2024-04-03T01:25:24.672140063Z ts=2024-04-03T01:25:24.672Z caller=main.go:1048 level=info msg="TSDB started"
2024-04-03T01:25:24.672157192Z ts=2024-04-03T01:25:24.672Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:25:24.674974491Z ts=2024-04-03T01:25:24.674Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.756491ms db_storage=2.497µs remote_storage=2.038µs web_handler=1.193µs query_engine=1.923µs scrape=645.04µs scrape_sd=49.117µs notify=43.998µs notify_sd=13.393µs rules=1.619279ms tracing=9.075µs
2024-04-03T01:25:24.674990253Z ts=2024-04-03T01:25:24.674Z caller=main.go:1009 level=info msg="Server is ready to receive web requests."
2024-04-03T01:25:24.674997851Z ts=2024-04-03T01:25:24.674Z caller=manager.go:1009 level=info component="rule manager" msg="Starting rule manager..."
2024-04-03T01:26:11.361773300Z ts=2024-04-03T01:26:11.361Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:26:11.364339037Z ts=2024-04-03T01:26:11.364Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.659782ms db_storage=10.155µs remote_storage=1.853µs web_handler=3.771µs query_engine=1.209µs scrape=233.926µs scrape_sd=55.706µs notify=64.074µs notify_sd=7.969µs rules=1.37072ms tracing=9.395µs
2024-04-03T01:40:47.503496548Z ts=2024-04-03T01:40:47.503Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:40:47.507480998Z ts=2024-04-03T01:40:47.507Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.031967ms db_storage=10.041µs remote_storage=1.789µs web_handler=5.105µs query_engine=1.066µs scrape=176.831µs scrape_sd=65.285µs notify=54.245µs notify_sd=7.776µs rules=2.789579ms tracing=14.272µs
2024-04-03T01:40:52.567068059Z ts=2024-04-03T01:40:52.566Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:40:52.568814851Z ts=2024-04-03T01:40:52.568Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.700759ms db_storage=1.883µs remote_storage=2.038µs web_handler=592ns query_engine=1.093µs scrape=78.252µs scrape_sd=23.317µs notify=78.041µs notify_sd=28.64µs rules=1.182683ms tracing=5.002µs
2024-04-03T01:42:17.099228903Z ts=2024-04-03T01:42:17.098Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:42:17.106424789Z ts=2024-04-03T01:42:17.106Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=7.361164ms db_storage=18.798µs remote_storage=2.671µs web_handler=8.551µs query_engine=1.742µs scrape=491.49µs scrape_sd=648.392µs notify=82.709µs notify_sd=16.675µs rules=2.930775ms tracing=18.582µs
2024-04-03T01:42:22.175460736Z ts=2024-04-03T01:42:22.175Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:42:22.177126400Z ts=2024-04-03T01:42:22.177Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.703755ms db_storage=2.039µs remote_storage=2.164µs web_handler=952ns query_engine=1.318µs scrape=123.878µs scrape_sd=42.26µs notify=22.091µs notify_sd=16.646µs rules=1.067418ms tracing=5.892µs
2024-04-03T01:43:43.881665986Z ts=2024-04-03T01:43:43.881Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:43:43.885021866Z ts=2024-04-03T01:43:43.884Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.413346ms db_storage=11.517µs remote_storage=1.703µs web_handler=4.973µs query_engine=1.383µs scrape=227.782µs scrape_sd=71.594µs notify=145.078µs notify_sd=10.047µs rules=2.120433ms tracing=13.422µs
2024-04-03T01:43:48.939435327Z ts=2024-04-03T01:43:48.939Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:43:48.941927867Z ts=2024-04-03T01:43:48.941Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.495762ms db_storage=2.344µs remote_storage=1.661µs web_handler=618ns query_engine=1.024µs scrape=94.744µs scrape_sd=27.103µs notify=18.382µs notify_sd=7.196µs rules=1.967703ms tracing=6.003µs
2024-04-03T01:43:54.010167042Z ts=2024-04-03T01:43:54.009Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:43:54.013644151Z ts=2024-04-03T01:43:54.013Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.464205ms db_storage=2.039µs remote_storage=2.126µs web_handler=655ns query_engine=1.257µs scrape=809.058µs scrape_sd=36.178µs notify=20.424µs notify_sd=11.957µs rules=2.189587ms tracing=6.196µs
2024-04-03T01:43:59.081634333Z ts=2024-04-03T01:43:59.081Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:43:59.084168104Z ts=2024-04-03T01:43:59.084Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.523462ms db_storage=2.085µs remote_storage=1.759µs web_handler=706ns query_engine=1.334µs scrape=100.194µs scrape_sd=34.212µs notify=16.906µs notify_sd=8.379µs rules=1.997864ms tracing=4.513µs
2024-04-03T01:44:04.146733105Z ts=2024-04-03T01:44:04.146Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:44:04.149111335Z ts=2024-04-03T01:44:04.148Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.4423ms db_storage=2.028µs remote_storage=1.783µs web_handler=728ns query_engine=1.006µs scrape=87.293µs scrape_sd=39.383µs notify=16.887µs notify_sd=6.787µs rules=1.947918ms tracing=5.957µs
2024-04-03T01:44:09.211879861Z ts=2024-04-03T01:44:09.211Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:44:09.221403243Z ts=2024-04-03T01:44:09.221Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=9.467923ms db_storage=3.244µs remote_storage=3.618µs web_handler=1.049µs query_engine=1.765µs scrape=271.714µs scrape_sd=83.728µs notify=38.36µs notify_sd=22.635µs rules=8.232012ms tracing=14.845µs
2024-04-03T01:45:31.421398321Z ts=2024-04-03T01:45:31.421Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:31.425776064Z ts=2024-04-03T01:45:31.425Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.500541ms db_storage=15.884µs remote_storage=2.916µs web_handler=6.839µs query_engine=21.018µs scrape=433.192µs scrape_sd=147.417µs notify=188.947µs notify_sd=16.243µs rules=2.284876ms tracing=39.262µs
2024-04-03T01:45:36.463122866Z ts=2024-04-03T01:45:36.462Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:36.473364921Z ts=2024-04-03T01:45:36.473Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=10.285319ms db_storage=2.989µs remote_storage=3.49µs web_handler=1.193µs query_engine=17.021µs scrape=234.185µs scrape_sd=97.92µs notify=64.189µs notify_sd=17.177µs rules=9.087919ms tracing=14.312µs
2024-04-03T01:45:41.510952894Z ts=2024-04-03T01:45:41.510Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:41.513491111Z ts=2024-04-03T01:45:41.513Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.540447ms db_storage=6.824µs remote_storage=11.559µs web_handler=853ns query_engine=24.543µs scrape=184.421µs scrape_sd=50.734µs notify=38.031µs notify_sd=9.479µs rules=1.515181ms tracing=6.749µs
2024-04-03T01:45:46.563290462Z ts=2024-04-03T01:45:46.563Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:46.566324601Z ts=2024-04-03T01:45:46.566Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.163509ms db_storage=10.058µs remote_storage=3.242µs web_handler=7.287µs query_engine=1.306µs scrape=238.394µs scrape_sd=65.57µs notify=165.246µs notify_sd=15.629µs rules=1.788774ms tracing=8.43µs
2024-04-03T01:45:51.601076990Z ts=2024-04-03T01:45:51.600Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:51.603801407Z ts=2024-04-03T01:45:51.603Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.242974ms db_storage=1.904µs remote_storage=2.057µs web_handler=507ns query_engine=1.359µs scrape=105.302µs scrape_sd=24.627µs notify=21.519µs notify_sd=7.595µs rules=857.182µs tracing=16.335µs
2024-04-03T01:45:56.654736111Z ts=2024-04-03T01:45:56.653Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T01:45:56.658444960Z ts=2024-04-03T01:45:56.657Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.932623ms db_storage=1.687µs remote_storage=1.798µs web_handler=549ns query_engine=999ns scrape=122µs scrape_sd=23.929µs notify=19.246µs notify_sd=22.905µs rules=2.707662ms tracing=8.139µs
2024-04-03T02:46:05.554735290Z ts=2024-04-03T02:46:05.554Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T02:46:05.560132607Z ts=2024-04-03T02:46:05.559Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=5.448152ms db_storage=14.686µs remote_storage=2.594µs web_handler=3.746µs query_engine=8.005µs scrape=251.691µs scrape_sd=79.413µs notify=66.779µs notify_sd=9.462µs rules=1.540027ms tracing=13.331µs
2024-04-03T02:48:32.586547937Z ts=2024-04-03T02:48:32.585Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"974b718c-302d-4f6b-8311-f2678d269b71\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"12150ba9-5f4a-4939-b0d6-49a708054905\", vdu_name=\"heal_04-autoheal-basic-1-autoheal_vnfd-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\", vnf_id=\"b7517d5e-ee01-42b3-b9cd-28ddae97d425\", vnf_member_index=\"autoheal-basic-1\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"974b718c-302d-4f6b-8311-f2678d269b71\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"12150ba9-5f4a-4939-b0d6-49a708054905\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\", vnf_id=\"b7517d5e-ee01-42b3-b9cd-28ddae97d425\", vnf_member_index=\"autoheal-basic-1\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T02:48:47.588775428Z ts=2024-04-03T02:48:47.587Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"974b718c-302d-4f6b-8311-f2678d269b71\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"12150ba9-5f4a-4939-b0d6-49a708054905\", vdu_name=\"heal_04-autoheal-basic-1-autoheal_vnfd-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\", vnf_id=\"b7517d5e-ee01-42b3-b9cd-28ddae97d425\", vnf_member_index=\"autoheal-basic-1\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"974b718c-302d-4f6b-8311-f2678d269b71\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"12150ba9-5f4a-4939-b0d6-49a708054905\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4d519def-9560-42a5-997a-f4df914be1c6\", vnf_id=\"b7517d5e-ee01-42b3-b9cd-28ddae97d425\", vnf_member_index=\"autoheal-basic-1\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T02:53:24.297630774Z ts=2024-04-03T02:53:24.297Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T02:53:24.301660259Z ts=2024-04-03T02:53:24.301Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=4.13721ms db_storage=16.788µs remote_storage=2.907µs web_handler=7.48µs query_engine=1.678µs scrape=258.887µs scrape_sd=98.121µs notify=79.978µs notify_sd=11.046µs rules=2.205381ms tracing=17.541µs
2024-04-03T03:18:59.235147326Z ts=2024-04-03T03:18:59.230Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:18:59.236460129Z ts=2024-04-03T03:18:59.236Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=5.245096ms db_storage=16.907µs remote_storage=2.672µs web_handler=6.991µs query_engine=1.551µs scrape=248.637µs scrape_sd=83.72µs notify=81.761µs notify_sd=11.865µs rules=3.69472ms tracing=20.089µs
2024-04-03T03:19:04.312745738Z ts=2024-04-03T03:19:04.312Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:19:04.318615689Z ts=2024-04-03T03:19:04.317Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=5.091221ms db_storage=2.268µs remote_storage=2.574µs web_handler=797ns query_engine=1.962µs scrape=206.358µs scrape_sd=40.563µs notify=47.654µs notify_sd=10.498µs rules=3.713029ms tracing=8.243µs
2024-04-03T03:26:24.175371248Z ts=2024-04-03T03:26:24.175Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:26:24.177605623Z ts=2024-04-03T03:26:24.177Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.288877ms db_storage=11.704µs remote_storage=1.971µs web_handler=5.27µs query_engine=1.464µs scrape=158.503µs scrape_sd=51.647µs notify=58.362µs notify_sd=7.098µs rules=1.12691ms tracing=11.024µs
2024-04-03T03:26:29.214363867Z ts=2024-04-03T03:26:29.214Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:26:29.215844282Z ts=2024-04-03T03:26:29.215Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.499958ms db_storage=1.918µs remote_storage=1.817µs web_handler=782ns query_engine=1.055µs scrape=151.642µs scrape_sd=36.644µs notify=15.196µs notify_sd=7.627µs rules=969.028µs tracing=4.784µs
2024-04-03T03:27:14.505555276Z ts=2024-04-03T03:27:14.505Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:27:14.507810638Z ts=2024-04-03T03:27:14.507Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.284779ms db_storage=1.684µs remote_storage=1.711µs web_handler=542ns query_engine=1.263µs scrape=128.77µs scrape_sd=28.489µs notify=54.333µs notify_sd=7.893µs rules=1.689658ms tracing=5.793µs
2024-04-03T03:27:19.559075668Z ts=2024-04-03T03:27:19.558Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:27:19.561141338Z ts=2024-04-03T03:27:19.561Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.064957ms db_storage=1.823µs remote_storage=1.969µs web_handler=568ns query_engine=951ns scrape=96.235µs scrape_sd=27.042µs notify=45.968µs notify_sd=9.42µs rules=1.496681ms tracing=5.563µs
2024-04-03T03:44:32.586247296Z ts=2024-04-03T03:44:32.585Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"aa5dac12-c347-43c8-b4ce-bfa740b9f1b3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"01f878d5-e6ce-4c8d-afa6-308941e49f70\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-1\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\", vnf_id=\"01905a62-8f5f-4d0a-afe6-e562a2981409\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"aa5dac12-c347-43c8-b4ce-bfa740b9f1b3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"01f878d5-e6ce-4c8d-afa6-308941e49f70\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\", vnf_id=\"01905a62-8f5f-4d0a-afe6-e562a2981409\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T03:44:47.521201839Z ts=2024-04-03T03:44:47.520Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:44:47.524099050Z ts=2024-04-03T03:44:47.523Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.936832ms db_storage=14.032µs remote_storage=2.231µs web_handler=5.431µs query_engine=1.241µs scrape=217.64µs scrape_sd=75.594µs notify=82.487µs notify_sd=11.773µs rules=1.43966ms tracing=13.124µs
2024-04-03T03:44:47.585438603Z ts=2024-04-03T03:44:47.585Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"aa5dac12-c347-43c8-b4ce-bfa740b9f1b3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"01f878d5-e6ce-4c8d-afa6-308941e49f70\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-1\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\", vnf_id=\"01905a62-8f5f-4d0a-afe6-e562a2981409\", vnf_member_index=\"vnf\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"aa5dac12-c347-43c8-b4ce-bfa740b9f1b3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"01f878d5-e6ce-4c8d-afa6-308941e49f70\", vdu_name=\"sa_02-vnf-hackfest_basic_metrics-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"4fccc44e-f3d2-4c66-91f8-113cbfdb7693\", vnf_id=\"01905a62-8f5f-4d0a-afe6-e562a2981409\", vnf_member_index=\"vnf\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T03:44:52.565186838Z ts=2024-04-03T03:44:52.565Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:44:52.569033486Z ts=2024-04-03T03:44:52.568Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.848738ms db_storage=3.156µs remote_storage=3.64µs web_handler=1.046µs query_engine=2.289µs scrape=222.427µs scrape_sd=70.17µs notify=231.655µs notify_sd=106.553µs rules=2.405085ms tracing=11.96µs
2024-04-03T03:47:04.587210625Z ts=2024-04-03T03:47:04.586Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:47:04.591106195Z ts=2024-04-03T03:47:04.590Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.959484ms db_storage=15.981µs remote_storage=6.933µs web_handler=7.318µs query_engine=2.159µs scrape=299.075µs scrape_sd=140.201µs notify=90.729µs notify_sd=13.636µs rules=2.28895ms tracing=20.723µs
2024-04-03T03:57:13.358581384Z ts=2024-04-03T03:57:13.358Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:57:13.361572945Z ts=2024-04-03T03:57:13.361Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=3.064806ms db_storage=10.605µs remote_storage=2.532µs web_handler=4.832µs query_engine=1.812µs scrape=248.107µs scrape_sd=77.389µs notify=78.816µs notify_sd=8.949µs rules=1.611106ms tracing=12.215µs
2024-04-03T03:59:34.263999409Z ts=2024-04-03T03:59:34.262Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T03:59:34.271381433Z ts=2024-04-03T03:59:34.271Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=8.582293ms db_storage=41.824µs remote_storage=2.915µs web_handler=7.638µs query_engine=1.556µs scrape=409.188µs scrape_sd=246.516µs notify=81.867µs notify_sd=23.458µs rules=2.797464ms tracing=16.874µs
2024-04-03T04:00:47.152492109Z ts=2024-04-03T04:00:47.152Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T04:00:47.155007661Z ts=2024-04-03T04:00:47.154Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.564331ms db_storage=10.318µs remote_storage=1.547µs web_handler=5.509µs query_engine=1.396µs scrape=378.955µs scrape_sd=53.249µs notify=78.056µs notify_sd=14.689µs rules=1.28728ms tracing=11.771µs
2024-04-03T04:04:32.585611692Z ts=2024-04-03T04:04:32.584Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"5654008b-6823-4c71-9897-61e8838e301b\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"ee6e7d3d-74ef-43f8-872a-dab4fbcb9e1b\", vdu_name=\"slicebasic.slice-edge-ubuntu_slice-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\", vnf_id=\"846ec7ad-cb6e-4a66-88a7-2d4eb1a09b35\", vnf_member_index=\"edge\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"5654008b-6823-4c71-9897-61e8838e301b\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"ee6e7d3d-74ef-43f8-872a-dab4fbcb9e1b\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\", vnf_id=\"846ec7ad-cb6e-4a66-88a7-2d4eb1a09b35\", vnf_member_index=\"edge\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T04:04:47.585294372Z ts=2024-04-03T04:04:47.584Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"5654008b-6823-4c71-9897-61e8838e301b\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"ee6e7d3d-74ef-43f8-872a-dab4fbcb9e1b\", vdu_name=\"slicebasic.slice-edge-ubuntu_slice-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\", vnf_id=\"846ec7ad-cb6e-4a66-88a7-2d4eb1a09b35\", vnf_member_index=\"edge\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"5654008b-6823-4c71-9897-61e8838e301b\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"ee6e7d3d-74ef-43f8-872a-dab4fbcb9e1b\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"bc9a50cd-b11f-47ed-9b3e-07351028d300\", vnf_id=\"846ec7ad-cb6e-4a66-88a7-2d4eb1a09b35\", vnf_member_index=\"edge\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T04:14:14.914473118Z ts=2024-04-03T04:14:14.914Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T04:14:14.921899098Z ts=2024-04-03T04:14:14.921Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=7.557444ms db_storage=23.472µs remote_storage=10.22µs web_handler=10.645µs query_engine=2.039µs scrape=459.172µs scrape_sd=114.012µs notify=100.319µs notify_sd=18.366µs rules=4.946354ms tracing=31.842µs
2024-04-03T04:14:19.974041452Z ts=2024-04-03T04:14:19.973Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T04:14:19.976592207Z ts=2024-04-03T04:14:19.976Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.630918ms db_storage=10.723µs remote_storage=2.467µs web_handler=6.7µs query_engine=1.569µs scrape=175.645µs scrape_sd=41.669µs notify=72.632µs notify_sd=19.487µs rules=1.641255ms tracing=7.941µs
2024-04-03T04:16:32.585612241Z ts=2024-04-03T04:16:32.584Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0f5dc377-1eee-4aee-ab5e-5a80b710e1e3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"26c1bc59-3cb4-46a2-8e3b-1fd7cce6069e\", vdu_name=\"sol003-instance--1-hackfest_basic_metrics-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\", vnf_id=\"e16a9fdd-57c2-48d8-96ed-4c303c47118d\", vnf_member_index=\"1\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0f5dc377-1eee-4aee-ab5e-5a80b710e1e3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"26c1bc59-3cb4-46a2-8e3b-1fd7cce6069e\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\", vnf_id=\"e16a9fdd-57c2-48d8-96ed-4c303c47118d\", vnf_member_index=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T04:16:47.584435536Z ts=2024-04-03T04:16:47.583Z caller=manager.go:677 level=warn component="rule manager" file=/etc/prometheus/osm_metric_rules.yml group=osm_metric_rules name=vm_status_extended index=0 msg="Evaluating rule failed" rule="record: vm_status_extended\nexpr: (last_over_time(vm_status[1m]) * on (vm_id, vim_id) group_left (ns_id, vnf_id,\n  vdu_id, project_id, job, vdu_name, vnf_member_index) last_over_time(ns_topology[1m]))\n  or (last_over_time(ns_topology[1m]) * -1)\nlabels:\n  job: osm_prometheus\n" err="found duplicate series for the match group {vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\"} on the right hand-side of the operation: [{__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0f5dc377-1eee-4aee-ab5e-5a80b710e1e3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"26c1bc59-3cb4-46a2-8e3b-1fd7cce6069e\", vdu_name=\"sol003-instance--1-hackfest_basic_metrics-VM-0\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\", vnf_id=\"e16a9fdd-57c2-48d8-96ed-4c303c47118d\", vnf_member_index=\"1\"}, {__name__=\"ns_topology\", job=\"airflow_osm_ns_topology\", ns_id=\"0f5dc377-1eee-4aee-ab5e-5a80b710e1e3\", project_id=\"906e76e1-3d4d-405e-9aef-801e4573a3ab\", vdu_id=\"26c1bc59-3cb4-46a2-8e3b-1fd7cce6069e\", vdu_name=\"UNKNOWN\", vim_id=\"7a8df917-c55f-4ae7-a49d-b123d795de17\", vm_id=\"f66a3ba1-a127-464c-8584-291b14a1e797\", vnf_id=\"e16a9fdd-57c2-48d8-96ed-4c303c47118d\", vnf_member_index=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side"
2024-04-03T04:20:02.186231194Z ts=2024-04-03T04:20:02.186Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T04:20:02.188735057Z ts=2024-04-03T04:20:02.188Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=2.557793ms db_storage=12.825µs remote_storage=1.849µs web_handler=5.366µs query_engine=1.381µs scrape=195.204µs scrape_sd=56.621µs notify=56.063µs notify_sd=8.851µs rules=1.266189ms tracing=10.914µs
2024-04-03T04:20:07.223261627Z ts=2024-04-03T04:20:07.223Z caller=main.go:1229 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2024-04-03T04:20:07.225003107Z ts=2024-04-03T04:20:07.224Z caller=main.go:1266 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.784299ms db_storage=7.55µs remote_storage=3.969µs web_handler=556ns query_engine=989ns scrape=126.458µs scrape_sd=49.802µs notify=18.692µs notify_sd=28.599µs rules=1.05153ms tracing=8.251µs
2024-04-03T01:26:06.236570613Z Created MongoClient to connect to MongoDB!
2024-04-03T01:26:06.236697614Z Generating prometheus config files
2024-04-03T01:26:06.305108261Z Jobs detected: 0
2024-04-03T01:26:06.309115266Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:26:06.311190148Z alerting:
2024-04-03T01:26:06.311217342Z   alertmanagers:
2024-04-03T01:26:06.311221298Z   - static_configs:
2024-04-03T01:26:06.311224405Z     - targets:
2024-04-03T01:26:06.311228094Z       - alertmanager:9093
2024-04-03T01:26:06.311231112Z global:
2024-04-03T01:26:06.311234205Z   evaluation_interval: 15s
2024-04-03T01:26:06.311237312Z   scrape_interval: 15s
2024-04-03T01:26:06.311240627Z rule_files:
2024-04-03T01:26:06.311243664Z - osm_metric_rules.yml
2024-04-03T01:26:06.311246665Z - osm_alert_rules.yml
2024-04-03T01:26:06.311249621Z scrape_configs:
2024-04-03T01:26:06.311252714Z - job_name: mon_exporter
2024-04-03T01:26:06.311255751Z   static_configs:
2024-04-03T01:26:06.311258714Z   - targets:
2024-04-03T01:26:06.311261843Z     - mon:8000
2024-04-03T01:26:06.311264883Z - honor_labels: true
2024-04-03T01:26:06.311267825Z   job_name: pushgateway
2024-04-03T01:26:06.311270905Z   scrape_interval: 30s
2024-04-03T01:26:06.311273957Z   static_configs:
2024-04-03T01:26:06.311276925Z   - targets:
2024-04-03T01:26:06.311281150Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:26:06.311284159Z 
2024-04-03T01:26:06.326685569Z Alerts read: 0
2024-04-03T01:26:06.326701674Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:26:06.326704984Z groups:
2024-04-03T01:26:06.326708679Z - name: osm_alert_rules
2024-04-03T01:26:06.326711661Z   rules:
2024-04-03T01:26:06.326722339Z   - alert: vdu_down
2024-04-03T01:26:06.326725748Z     annotations:
2024-04-03T01:26:06.326730351Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:26:06.326734233Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:26:06.326737315Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:26:06.326740857Z     expr: vm_status_extended != 1
2024-04-03T01:26:06.326743803Z     for: 3m
2024-04-03T01:26:06.326746598Z 
2024-04-03T01:26:06.326749573Z New config written, updating prometheus
2024-04-03T01:26:06.341099291Z Error in configuration attempt! Number of tries: 1/3
2024-04-03T01:26:06.341117085Z Cannot connect to host prometheus:9090 ssl:default [Connect call failed ('10.103.43.242', 9090)]
2024-04-03T01:26:11.342698895Z Generating prometheus config files
2024-04-03T01:26:11.342715694Z Jobs detected: 0
2024-04-03T01:26:11.348793365Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:26:11.348810561Z alerting:
2024-04-03T01:26:11.348814768Z   alertmanagers:
2024-04-03T01:26:11.348850082Z   - static_configs:
2024-04-03T01:26:11.348854400Z     - targets:
2024-04-03T01:26:11.348858231Z       - alertmanager:9093
2024-04-03T01:26:11.348861309Z global:
2024-04-03T01:26:11.348864307Z   evaluation_interval: 15s
2024-04-03T01:26:11.348867656Z   scrape_interval: 15s
2024-04-03T01:26:11.348870748Z rule_files:
2024-04-03T01:26:11.348873984Z - osm_metric_rules.yml
2024-04-03T01:26:11.348876942Z - osm_alert_rules.yml
2024-04-03T01:26:11.348879928Z scrape_configs:
2024-04-03T01:26:11.348886505Z - job_name: mon_exporter
2024-04-03T01:26:11.348889589Z   static_configs:
2024-04-03T01:26:11.348892626Z   - targets:
2024-04-03T01:26:11.348895687Z     - mon:8000
2024-04-03T01:26:11.348898613Z - honor_labels: true
2024-04-03T01:26:11.348916703Z   job_name: pushgateway
2024-04-03T01:26:11.348919835Z   scrape_interval: 30s
2024-04-03T01:26:11.348922766Z   static_configs:
2024-04-03T01:26:11.348925771Z   - targets:
2024-04-03T01:26:11.348929277Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:26:11.348932180Z 
2024-04-03T01:26:11.353985728Z Alerts read: 0
2024-04-03T01:26:11.354795746Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:26:11.356516273Z groups:
2024-04-03T01:26:11.356528663Z - name: osm_alert_rules
2024-04-03T01:26:11.356532099Z   rules:
2024-04-03T01:26:11.356535380Z   - alert: vdu_down
2024-04-03T01:26:11.356538485Z     annotations:
2024-04-03T01:26:11.356542543Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:26:11.356546251Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:26:11.356549637Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:26:11.356553374Z     expr: vm_status_extended != 1
2024-04-03T01:26:11.356556396Z     for: 3m
2024-04-03T01:26:11.356559223Z 
2024-04-03T01:26:11.357534345Z New config written, updating prometheus
2024-04-03T01:26:16.398774098Z Prometheus config update successful
2024-04-03T01:26:21.421666660Z Listening to changes in prometheus jobs and alerts collections
2024-04-03T01:40:47.469763851Z Changes detected, updating prometheus config
2024-04-03T01:40:47.471801832Z Jobs detected: 0
2024-04-03T01:40:47.479287481Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:40:47.482063816Z alerting:
2024-04-03T01:40:47.482078507Z   alertmanagers:
2024-04-03T01:40:47.482084036Z   - static_configs:
2024-04-03T01:40:47.482088262Z     - targets:
2024-04-03T01:40:47.482092553Z       - alertmanager:9093
2024-04-03T01:40:47.482096242Z global:
2024-04-03T01:40:47.482100053Z   evaluation_interval: 15s
2024-04-03T01:40:47.482103695Z   scrape_interval: 15s
2024-04-03T01:40:47.482107490Z rule_files:
2024-04-03T01:40:47.482111242Z - osm_metric_rules.yml
2024-04-03T01:40:47.482114972Z - osm_alert_rules.yml
2024-04-03T01:40:47.482118716Z scrape_configs:
2024-04-03T01:40:47.482122385Z - job_name: mon_exporter
2024-04-03T01:40:47.482126048Z   static_configs:
2024-04-03T01:40:47.482129913Z   - targets:
2024-04-03T01:40:47.482133629Z     - mon:8000
2024-04-03T01:40:47.482137313Z - honor_labels: true
2024-04-03T01:40:47.482141103Z   job_name: pushgateway
2024-04-03T01:40:47.482144833Z   scrape_interval: 30s
2024-04-03T01:40:47.482148449Z   static_configs:
2024-04-03T01:40:47.482152212Z   - targets:
2024-04-03T01:40:47.482156703Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:40:47.482227889Z 
2024-04-03T01:40:47.487818663Z Alerts read: 2
2024-04-03T01:40:47.490982955Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:40:47.494546831Z groups:
2024-04-03T01:40:47.494557014Z - name: osm_alert_rules
2024-04-03T01:40:47.494560092Z   rules:
2024-04-03T01:40:47.494563126Z   - alert: vdu_down
2024-04-03T01:40:47.494566247Z     annotations:
2024-04-03T01:40:47.494569991Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:40:47.494573319Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:40:47.494576121Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:40:47.494579503Z     expr: vm_status_extended != 1
2024-04-03T01:40:47.494582306Z     for: 3m
2024-04-03T01:40:47.494623296Z - name: _osm_alert_rules_20240403014047_
2024-04-03T01:40:47.494627329Z   rules:
2024-04-03T01:40:47.494630854Z   - alert: scalein_3f8f7fd3-ce89-4446-acf5-e137d0ab39d1
2024-04-03T01:40:47.494633882Z     expr: (count (osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:47.494637351Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:47.494640183Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:40:47.494643000Z     for: 1m
2024-04-03T01:40:47.494645796Z     labels:
2024-04-03T01:40:47.494648581Z       ns_id: 7b042bbb-5b96-4afb-babf-9a82e5f31bdc
2024-04-03T01:40:47.494651346Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:40:47.494654997Z       vnf_member_index: vnf
2024-04-03T01:40:47.494657910Z   - alert: scaleout_0a6ee798-af4d-4070-8543-2192ae3e1102
2024-04-03T01:40:47.494660674Z     expr: (count (osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:47.494663425Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:47.494666204Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:40:47.494669511Z     for: 1m
2024-04-03T01:40:47.494672237Z     labels:
2024-04-03T01:40:47.494675134Z       ns_id: 7b042bbb-5b96-4afb-babf-9a82e5f31bdc
2024-04-03T01:40:47.494677893Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:40:47.494680696Z       vnf_member_index: vnf
2024-04-03T01:40:47.494695033Z 
2024-04-03T01:40:47.497764755Z New config written, updating prometheus
2024-04-03T01:40:52.535304518Z Prometheus config update successful
2024-04-03T01:40:52.535378808Z 
2024-04-03T01:40:52.536798040Z Changes detected, updating prometheus config
2024-04-03T01:40:52.537822117Z Jobs detected: 0
2024-04-03T01:40:52.543657005Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:40:52.545714785Z alerting:
2024-04-03T01:40:52.545750860Z   alertmanagers:
2024-04-03T01:40:52.545761548Z   - static_configs:
2024-04-03T01:40:52.545770613Z     - targets:
2024-04-03T01:40:52.545779659Z       - alertmanager:9093
2024-04-03T01:40:52.545788034Z global:
2024-04-03T01:40:52.545796974Z   evaluation_interval: 15s
2024-04-03T01:40:52.545805494Z   scrape_interval: 15s
2024-04-03T01:40:52.545813871Z rule_files:
2024-04-03T01:40:52.545822242Z - osm_metric_rules.yml
2024-04-03T01:40:52.545830505Z - osm_alert_rules.yml
2024-04-03T01:40:52.545838756Z scrape_configs:
2024-04-03T01:40:52.545847890Z - job_name: mon_exporter
2024-04-03T01:40:52.545856316Z   static_configs:
2024-04-03T01:40:52.545864810Z   - targets:
2024-04-03T01:40:52.545873257Z     - mon:8000
2024-04-03T01:40:52.545881672Z - honor_labels: true
2024-04-03T01:40:52.545890028Z   job_name: pushgateway
2024-04-03T01:40:52.545898228Z   scrape_interval: 30s
2024-04-03T01:40:52.545906544Z   static_configs:
2024-04-03T01:40:52.545914848Z   - targets:
2024-04-03T01:40:52.545924326Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:40:52.545950129Z 
2024-04-03T01:40:52.550070489Z Alerts read: 2
2024-04-03T01:40:52.551954112Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:40:52.556145959Z groups:
2024-04-03T01:40:52.556176045Z - name: osm_alert_rules
2024-04-03T01:40:52.556185809Z   rules:
2024-04-03T01:40:52.556194891Z   - alert: vdu_down
2024-04-03T01:40:52.556229558Z     annotations:
2024-04-03T01:40:52.556239972Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:40:52.556249422Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:40:52.556257783Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:40:52.556267337Z     expr: vm_status_extended != 1
2024-04-03T01:40:52.556276117Z     for: 3m
2024-04-03T01:40:52.556284984Z - name: _osm_alert_rules_20240403014052_
2024-04-03T01:40:52.556293456Z   rules:
2024-04-03T01:40:52.556314321Z   - alert: scalein_3f8f7fd3-ce89-4446-acf5-e137d0ab39d1
2024-04-03T01:40:52.556323092Z     expr: (count (osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:52.556332966Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:52.556341551Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:40:52.556350341Z     for: 1m
2024-04-03T01:40:52.556358814Z     labels:
2024-04-03T01:40:52.556367287Z       ns_id: 7b042bbb-5b96-4afb-babf-9a82e5f31bdc
2024-04-03T01:40:52.556375664Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:40:52.556385668Z       vnf_member_index: vnf
2024-04-03T01:40:52.556394122Z   - alert: scaleout_0a6ee798-af4d-4070-8543-2192ae3e1102
2024-04-03T01:40:52.556402766Z     expr: (count (osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:52.556411323Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="7b042bbb-5b96-4afb-babf-9a82e5f31bdc",
2024-04-03T01:40:52.556420821Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:40:52.556429332Z     for: 1m
2024-04-03T01:40:52.556437883Z     labels:
2024-04-03T01:40:52.556446304Z       ns_id: 7b042bbb-5b96-4afb-babf-9a82e5f31bdc
2024-04-03T01:40:52.556454699Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:40:52.556463285Z       vnf_member_index: vnf
2024-04-03T01:40:52.556478879Z 
2024-04-03T01:40:52.560173327Z New config written, updating prometheus
2024-04-03T01:40:57.589243630Z Prometheus config update successful
2024-04-03T01:40:57.589277790Z 
2024-04-03T01:42:17.064387315Z Changes detected, updating prometheus config
2024-04-03T01:42:17.066495222Z Jobs detected: 0
2024-04-03T01:42:17.077224207Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:42:17.080701490Z alerting:
2024-04-03T01:42:17.080719526Z   alertmanagers:
2024-04-03T01:42:17.080725182Z   - static_configs:
2024-04-03T01:42:17.080729666Z     - targets:
2024-04-03T01:42:17.080734547Z       - alertmanager:9093
2024-04-03T01:42:17.080739570Z global:
2024-04-03T01:42:17.080744003Z   evaluation_interval: 15s
2024-04-03T01:42:17.080748118Z   scrape_interval: 15s
2024-04-03T01:42:17.080752226Z rule_files:
2024-04-03T01:42:17.080756240Z - osm_metric_rules.yml
2024-04-03T01:42:17.080760051Z - osm_alert_rules.yml
2024-04-03T01:42:17.080764627Z scrape_configs:
2024-04-03T01:42:17.080768547Z - job_name: mon_exporter
2024-04-03T01:42:17.080772859Z   static_configs:
2024-04-03T01:42:17.080777639Z   - targets:
2024-04-03T01:42:17.080781663Z     - mon:8000
2024-04-03T01:42:17.080786241Z - honor_labels: true
2024-04-03T01:42:17.080790479Z   job_name: pushgateway
2024-04-03T01:42:17.080794681Z   scrape_interval: 30s
2024-04-03T01:42:17.080798808Z   static_configs:
2024-04-03T01:42:17.080803467Z   - targets:
2024-04-03T01:42:17.080825420Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:42:17.080829624Z 
2024-04-03T01:42:17.085008457Z Alerts read: 0
2024-04-03T01:42:17.088463707Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:42:17.090751564Z groups:
2024-04-03T01:42:17.090772316Z - name: osm_alert_rules
2024-04-03T01:42:17.090778740Z   rules:
2024-04-03T01:42:17.090784270Z   - alert: vdu_down
2024-04-03T01:42:17.090789663Z     annotations:
2024-04-03T01:42:17.090801117Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:42:17.090807039Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:42:17.090812238Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:42:17.090817757Z     expr: vm_status_extended != 1
2024-04-03T01:42:17.090823223Z     for: 3m
2024-04-03T01:42:17.090828265Z 
2024-04-03T01:42:17.092578125Z New config written, updating prometheus
2024-04-03T01:42:22.132234968Z Prometheus config update successful
2024-04-03T01:42:22.132274126Z 
2024-04-03T01:42:22.132531178Z Changes detected, updating prometheus config
2024-04-03T01:42:22.134425475Z Jobs detected: 0
2024-04-03T01:42:22.144669650Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:42:22.150154562Z alerting:
2024-04-03T01:42:22.150181193Z   alertmanagers:
2024-04-03T01:42:22.150187092Z   - static_configs:
2024-04-03T01:42:22.150191598Z     - targets:
2024-04-03T01:42:22.150196306Z       - alertmanager:9093
2024-04-03T01:42:22.150201283Z global:
2024-04-03T01:42:22.150205468Z   evaluation_interval: 15s
2024-04-03T01:42:22.150209500Z   scrape_interval: 15s
2024-04-03T01:42:22.150213746Z rule_files:
2024-04-03T01:42:22.150217810Z - osm_metric_rules.yml
2024-04-03T01:42:22.150221924Z - osm_alert_rules.yml
2024-04-03T01:42:22.150226029Z scrape_configs:
2024-04-03T01:42:22.150230060Z - job_name: mon_exporter
2024-04-03T01:42:22.150234039Z   static_configs:
2024-04-03T01:42:22.150238234Z   - targets:
2024-04-03T01:42:22.150242589Z     - mon:8000
2024-04-03T01:42:22.150246596Z - honor_labels: true
2024-04-03T01:42:22.150250553Z   job_name: pushgateway
2024-04-03T01:42:22.150254665Z   scrape_interval: 30s
2024-04-03T01:42:22.150258943Z   static_configs:
2024-04-03T01:42:22.150263160Z   - targets:
2024-04-03T01:42:22.150267808Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:42:22.150271743Z 
2024-04-03T01:42:22.157795132Z Alerts read: 0
2024-04-03T01:42:22.161577715Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:42:22.164591087Z groups:
2024-04-03T01:42:22.164606634Z - name: osm_alert_rules
2024-04-03T01:42:22.164610111Z   rules:
2024-04-03T01:42:22.164613175Z   - alert: vdu_down
2024-04-03T01:42:22.164616282Z     annotations:
2024-04-03T01:42:22.164620295Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:42:22.164626818Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:42:22.164629886Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:42:22.164633423Z     expr: vm_status_extended != 1
2024-04-03T01:42:22.164636499Z     for: 3m
2024-04-03T01:42:22.164639336Z 
2024-04-03T01:42:22.167095551Z New config written, updating prometheus
2024-04-03T01:42:27.189237689Z Prometheus config update successful
2024-04-03T01:42:27.189276842Z 
2024-04-03T01:43:43.850283023Z Changes detected, updating prometheus config
2024-04-03T01:43:43.851923775Z Jobs detected: 0
2024-04-03T01:43:43.857013689Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:43:43.859220536Z alerting:
2024-04-03T01:43:43.859231875Z   alertmanagers:
2024-04-03T01:43:43.859235528Z   - static_configs:
2024-04-03T01:43:43.859238591Z     - targets:
2024-04-03T01:43:43.859242193Z       - alertmanager:9093
2024-04-03T01:43:43.859245126Z global:
2024-04-03T01:43:43.859248140Z   evaluation_interval: 15s
2024-04-03T01:43:43.859251272Z   scrape_interval: 15s
2024-04-03T01:43:43.859254257Z rule_files:
2024-04-03T01:43:43.859257303Z - osm_metric_rules.yml
2024-04-03T01:43:43.859260224Z - osm_alert_rules.yml
2024-04-03T01:43:43.859263768Z scrape_configs:
2024-04-03T01:43:43.859266849Z - job_name: mon_exporter
2024-04-03T01:43:43.859269796Z   static_configs:
2024-04-03T01:43:43.859272759Z   - targets:
2024-04-03T01:43:43.859275749Z     - mon:8000
2024-04-03T01:43:43.859278630Z - honor_labels: true
2024-04-03T01:43:43.859281583Z   job_name: pushgateway
2024-04-03T01:43:43.859284530Z   scrape_interval: 30s
2024-04-03T01:43:43.859287464Z   static_configs:
2024-04-03T01:43:43.859290315Z   - targets:
2024-04-03T01:43:43.859293887Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:43:43.859296774Z 
2024-04-03T01:43:43.864636056Z Alerts read: 2
2024-04-03T01:43:43.869051082Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:43:43.872952947Z groups:
2024-04-03T01:43:43.872976833Z - name: osm_alert_rules
2024-04-03T01:43:43.872980556Z   rules:
2024-04-03T01:43:43.872984697Z   - alert: vdu_down
2024-04-03T01:43:43.872987649Z     annotations:
2024-04-03T01:43:43.872991406Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:43:43.872994850Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:43:43.872997832Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:43:43.873001307Z     expr: vm_status_extended != 1
2024-04-03T01:43:43.873004315Z     for: 3m
2024-04-03T01:43:43.873007311Z - name: _osm_alert_rules_20240403014343_
2024-04-03T01:43:43.873010218Z   rules:
2024-04-03T01:43:43.873014193Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:43:43.873017193Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:43.873020940Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:43.873023935Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:43.873027413Z     for: 1m
2024-04-03T01:43:43.873030335Z     labels:
2024-04-03T01:43:43.873033313Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:43.873036636Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:43.873040063Z       vnf_member_index: vnf
2024-04-03T01:43:43.873043061Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:43:43.873045997Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:43.873049034Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:43.873051951Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:43.873054838Z     for: 1m
2024-04-03T01:43:43.873057796Z     labels:
2024-04-03T01:43:43.873082164Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:43.873085320Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:43.873088329Z       vnf_member_index: vnf
2024-04-03T01:43:43.873091090Z 
2024-04-03T01:43:43.877148523Z New config written, updating prometheus
2024-04-03T01:43:48.904357176Z Prometheus config update successful
2024-04-03T01:43:48.904404287Z 
2024-04-03T01:43:48.905809730Z Changes detected, updating prometheus config
2024-04-03T01:43:48.906976636Z Jobs detected: 0
2024-04-03T01:43:48.912625881Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:43:48.915259609Z alerting:
2024-04-03T01:43:48.915270992Z   alertmanagers:
2024-04-03T01:43:48.915274694Z   - static_configs:
2024-04-03T01:43:48.915277873Z     - targets:
2024-04-03T01:43:48.915281530Z       - alertmanager:9093
2024-04-03T01:43:48.915284604Z global:
2024-04-03T01:43:48.915287688Z   evaluation_interval: 15s
2024-04-03T01:43:48.915290725Z   scrape_interval: 15s
2024-04-03T01:43:48.915293979Z rule_files:
2024-04-03T01:43:48.915297910Z - osm_metric_rules.yml
2024-04-03T01:43:48.915300940Z - osm_alert_rules.yml
2024-04-03T01:43:48.915304095Z scrape_configs:
2024-04-03T01:43:48.915307062Z - job_name: mon_exporter
2024-04-03T01:43:48.915310091Z   static_configs:
2024-04-03T01:43:48.915313233Z   - targets:
2024-04-03T01:43:48.915316212Z     - mon:8000
2024-04-03T01:43:48.915319347Z - honor_labels: true
2024-04-03T01:43:48.915322395Z   job_name: pushgateway
2024-04-03T01:43:48.915329321Z   scrape_interval: 30s
2024-04-03T01:43:48.915332674Z   static_configs:
2024-04-03T01:43:48.915335675Z   - targets:
2024-04-03T01:43:48.915339351Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:43:48.915342280Z 
2024-04-03T01:43:48.919366871Z Alerts read: 6
2024-04-03T01:43:48.921514210Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:43:48.928659072Z groups:
2024-04-03T01:43:48.928676845Z - name: osm_alert_rules
2024-04-03T01:43:48.928681773Z   rules:
2024-04-03T01:43:48.928686211Z   - alert: vdu_down
2024-04-03T01:43:48.928690978Z     annotations:
2024-04-03T01:43:48.928696014Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:43:48.928701309Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:43:48.928705515Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:43:48.928710481Z     expr: vm_status_extended != 1
2024-04-03T01:43:48.928715383Z     for: 3m
2024-04-03T01:43:48.928719745Z - name: _osm_alert_rules_20240403014348_
2024-04-03T01:43:48.928724033Z   rules:
2024-04-03T01:43:48.928729320Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:43:48.928733642Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928738669Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928743353Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:48.928747905Z     for: 1m
2024-04-03T01:43:48.928752217Z     labels:
2024-04-03T01:43:48.928756581Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.928761299Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.928766337Z       vnf_member_index: vnf
2024-04-03T01:43:48.928770577Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:43:48.928775068Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928798264Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928802730Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:48.928807059Z     for: 1m
2024-04-03T01:43:48.928811341Z     labels:
2024-04-03T01:43:48.928815668Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.928820002Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.928824474Z       vnf_member_index: vnf
2024-04-03T01:43:48.928830827Z   - alert: scalein_7fefd11f-bbce-4b79-961a-7dd5ca411880
2024-04-03T01:43:48.928836749Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928842705Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928848664Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:48.928854798Z     for: 1m
2024-04-03T01:43:48.928860764Z     labels:
2024-04-03T01:43:48.928865946Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.928870380Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.928874493Z       vnf_member_index: vnf
2024-04-03T01:43:48.928878494Z   - alert: scaleout_3ddff1ef-8bd4-4425-ac5a-ee18c921f211
2024-04-03T01:43:48.928882518Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928886863Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928890860Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:48.928894821Z     for: 1m
2024-04-03T01:43:48.928898859Z     labels:
2024-04-03T01:43:48.928902821Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.928906777Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.928911694Z       vnf_member_index: vnf
2024-04-03T01:43:48.928918128Z   - alert: scalein_2664e90b-0c32-4aa2-88f7-8c41650bb41e
2024-04-03T01:43:48.928924580Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928930751Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928936258Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:48.928940560Z     for: 1m
2024-04-03T01:43:48.928944609Z     labels:
2024-04-03T01:43:48.928948767Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.928952792Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.928956767Z       vnf_member_index: vnf
2024-04-03T01:43:48.928960835Z   - alert: scaleout_66d78076-2600-47b9-b617-a3f4fc102bbf
2024-04-03T01:43:48.928964821Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928987747Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:48.928992523Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:48.928996661Z     for: 1m
2024-04-03T01:43:48.929000680Z     labels:
2024-04-03T01:43:48.929004672Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:48.929015502Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:48.929019599Z       vnf_member_index: vnf
2024-04-03T01:43:48.929023385Z 
2024-04-03T01:43:48.936297846Z New config written, updating prometheus
2024-04-03T01:43:53.974655651Z Prometheus config update successful
2024-04-03T01:43:53.974701741Z 
2024-04-03T01:43:53.974711876Z Changes detected, updating prometheus config
2024-04-03T01:43:53.976621647Z Jobs detected: 0
2024-04-03T01:43:53.982703411Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:43:53.984448927Z alerting:
2024-04-03T01:43:53.984457879Z   alertmanagers:
2024-04-03T01:43:53.984461995Z   - static_configs:
2024-04-03T01:43:53.984465178Z     - targets:
2024-04-03T01:43:53.984468811Z       - alertmanager:9093
2024-04-03T01:43:53.984472229Z global:
2024-04-03T01:43:53.984475558Z   evaluation_interval: 15s
2024-04-03T01:43:53.984478549Z   scrape_interval: 15s
2024-04-03T01:43:53.984481521Z rule_files:
2024-04-03T01:43:53.984484579Z - osm_metric_rules.yml
2024-04-03T01:43:53.984487455Z - osm_alert_rules.yml
2024-04-03T01:43:53.984491418Z scrape_configs:
2024-04-03T01:43:53.984495726Z - job_name: mon_exporter
2024-04-03T01:43:53.984500099Z   static_configs:
2024-04-03T01:43:53.984505220Z   - targets:
2024-04-03T01:43:53.984509953Z     - mon:8000
2024-04-03T01:43:53.984514510Z - honor_labels: true
2024-04-03T01:43:53.984519008Z   job_name: pushgateway
2024-04-03T01:43:53.984525884Z   scrape_interval: 30s
2024-04-03T01:43:53.984533295Z   static_configs:
2024-04-03T01:43:53.984538160Z   - targets:
2024-04-03T01:43:53.984543119Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:43:53.984547571Z 
2024-04-03T01:43:53.988597753Z Alerts read: 6
2024-04-03T01:43:53.992461944Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:43:53.999369677Z groups:
2024-04-03T01:43:53.999388517Z - name: osm_alert_rules
2024-04-03T01:43:53.999394699Z   rules:
2024-04-03T01:43:53.999399393Z   - alert: vdu_down
2024-04-03T01:43:53.999404864Z     annotations:
2024-04-03T01:43:53.999410484Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:43:53.999415650Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:43:53.999419848Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:43:53.999423694Z     expr: vm_status_extended != 1
2024-04-03T01:43:53.999426669Z     for: 3m
2024-04-03T01:43:53.999429753Z - name: _osm_alert_rules_20240403014353_
2024-04-03T01:43:53.999432715Z   rules:
2024-04-03T01:43:53.999436233Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:43:53.999439238Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999442782Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999445721Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:53.999448622Z     for: 1m
2024-04-03T01:43:53.999451472Z     labels:
2024-04-03T01:43:53.999454409Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999457272Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999460858Z       vnf_member_index: vnf
2024-04-03T01:43:53.999463790Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:43:53.999466718Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999482879Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999485957Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:53.999488818Z     for: 1m
2024-04-03T01:43:53.999492082Z     labels:
2024-04-03T01:43:53.999495192Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999498091Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999501015Z       vnf_member_index: vnf
2024-04-03T01:43:53.999503916Z   - alert: scalein_7fefd11f-bbce-4b79-961a-7dd5ca411880
2024-04-03T01:43:53.999506834Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999509875Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999512868Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:53.999515935Z     for: 1m
2024-04-03T01:43:53.999518789Z     labels:
2024-04-03T01:43:53.999521692Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999524849Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999527712Z       vnf_member_index: vnf
2024-04-03T01:43:53.999530634Z   - alert: scaleout_3ddff1ef-8bd4-4425-ac5a-ee18c921f211
2024-04-03T01:43:53.999533816Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999536904Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999539922Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:53.999542796Z     for: 1m
2024-04-03T01:43:53.999545648Z     labels:
2024-04-03T01:43:53.999548536Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999551445Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999554369Z       vnf_member_index: vnf
2024-04-03T01:43:53.999557251Z   - alert: scalein_2664e90b-0c32-4aa2-88f7-8c41650bb41e
2024-04-03T01:43:53.999560103Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999563051Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999565949Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:53.999568798Z     for: 1m
2024-04-03T01:43:53.999571662Z     labels:
2024-04-03T01:43:53.999574564Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999577428Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999580316Z       vnf_member_index: vnf
2024-04-03T01:43:53.999583198Z   - alert: scaleout_66d78076-2600-47b9-b617-a3f4fc102bbf
2024-04-03T01:43:53.999586079Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999589027Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:53.999591887Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:53.999594750Z     for: 1m
2024-04-03T01:43:53.999597730Z     labels:
2024-04-03T01:43:53.999600618Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:53.999603501Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:53.999610838Z       vnf_member_index: vnf
2024-04-03T01:43:53.999613785Z 
2024-04-03T01:43:54.007257790Z New config written, updating prometheus
2024-04-03T01:43:59.042882614Z Prometheus config update successful
2024-04-03T01:43:59.042902983Z 
2024-04-03T01:43:59.042907065Z Changes detected, updating prometheus config
2024-04-03T01:43:59.044301259Z Jobs detected: 0
2024-04-03T01:43:59.054366416Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:43:59.057100218Z alerting:
2024-04-03T01:43:59.057122900Z   alertmanagers:
2024-04-03T01:43:59.057126890Z   - static_configs:
2024-04-03T01:43:59.057130532Z     - targets:
2024-04-03T01:43:59.057134239Z       - alertmanager:9093
2024-04-03T01:43:59.057137404Z global:
2024-04-03T01:43:59.057140489Z   evaluation_interval: 15s
2024-04-03T01:43:59.057143576Z   scrape_interval: 15s
2024-04-03T01:43:59.057146700Z rule_files:
2024-04-03T01:43:59.057150035Z - osm_metric_rules.yml
2024-04-03T01:43:59.057153096Z - osm_alert_rules.yml
2024-04-03T01:43:59.057156533Z scrape_configs:
2024-04-03T01:43:59.057159734Z - job_name: mon_exporter
2024-04-03T01:43:59.057162757Z   static_configs:
2024-04-03T01:43:59.057166037Z   - targets:
2024-04-03T01:43:59.057169094Z     - mon:8000
2024-04-03T01:43:59.057172194Z - honor_labels: true
2024-04-03T01:43:59.057175354Z   job_name: pushgateway
2024-04-03T01:43:59.057178438Z   scrape_interval: 30s
2024-04-03T01:43:59.057181843Z   static_configs:
2024-04-03T01:43:59.057184924Z   - targets:
2024-04-03T01:43:59.057188778Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:43:59.057191913Z 
2024-04-03T01:43:59.061573827Z Alerts read: 6
2024-04-03T01:43:59.063363447Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:43:59.069594127Z groups:
2024-04-03T01:43:59.069611821Z - name: osm_alert_rules
2024-04-03T01:43:59.069616552Z   rules:
2024-04-03T01:43:59.069620776Z   - alert: vdu_down
2024-04-03T01:43:59.069624965Z     annotations:
2024-04-03T01:43:59.069629988Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:43:59.069664320Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:43:59.069669372Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:43:59.069674145Z     expr: vm_status_extended != 1
2024-04-03T01:43:59.069678354Z     for: 3m
2024-04-03T01:43:59.069682449Z - name: _osm_alert_rules_20240403014359_
2024-04-03T01:43:59.069687169Z   rules:
2024-04-03T01:43:59.069691894Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:43:59.069695997Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069700902Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069705041Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:59.069709197Z     for: 1m
2024-04-03T01:43:59.069713276Z     labels:
2024-04-03T01:43:59.069717780Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069721816Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069726389Z       vnf_member_index: vnf
2024-04-03T01:43:59.069730555Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:43:59.069734930Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069739237Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069757335Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:59.069761579Z     for: 1m
2024-04-03T01:43:59.069765739Z     labels:
2024-04-03T01:43:59.069769821Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069773923Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069777907Z       vnf_member_index: vnf
2024-04-03T01:43:59.069781937Z   - alert: scalein_7fefd11f-bbce-4b79-961a-7dd5ca411880
2024-04-03T01:43:59.069786079Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069790234Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069794269Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:59.069798300Z     for: 1m
2024-04-03T01:43:59.069802363Z     labels:
2024-04-03T01:43:59.069810802Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069814949Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069819037Z       vnf_member_index: vnf
2024-04-03T01:43:59.069823222Z   - alert: scaleout_3ddff1ef-8bd4-4425-ac5a-ee18c921f211
2024-04-03T01:43:59.069827259Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069831579Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069835708Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:59.069839747Z     for: 1m
2024-04-03T01:43:59.069843843Z     labels:
2024-04-03T01:43:59.069847871Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069851912Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069855944Z       vnf_member_index: vnf
2024-04-03T01:43:59.069859986Z   - alert: scalein_2664e90b-0c32-4aa2-88f7-8c41650bb41e
2024-04-03T01:43:59.069864006Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069868110Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069872206Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:43:59.069876352Z     for: 1m
2024-04-03T01:43:59.069880445Z     labels:
2024-04-03T01:43:59.069884458Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069888587Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069892656Z       vnf_member_index: vnf
2024-04-03T01:43:59.069896663Z   - alert: scaleout_66d78076-2600-47b9-b617-a3f4fc102bbf
2024-04-03T01:43:59.069900720Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069904867Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:43:59.069908911Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:43:59.069912895Z     for: 1m
2024-04-03T01:43:59.069916964Z     labels:
2024-04-03T01:43:59.069921112Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:43:59.069927838Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:43:59.069932048Z       vnf_member_index: vnf
2024-04-03T01:43:59.069935954Z 
2024-04-03T01:43:59.079248976Z New config written, updating prometheus
2024-04-03T01:44:04.110440069Z Prometheus config update successful
2024-04-03T01:44:04.110613152Z 
2024-04-03T01:44:04.110626738Z Changes detected, updating prometheus config
2024-04-03T01:44:04.112632806Z Jobs detected: 0
2024-04-03T01:44:04.119122016Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:44:04.121496411Z alerting:
2024-04-03T01:44:04.121508657Z   alertmanagers:
2024-04-03T01:44:04.121512643Z   - static_configs:
2024-04-03T01:44:04.121552781Z     - targets:
2024-04-03T01:44:04.121558057Z       - alertmanager:9093
2024-04-03T01:44:04.121562226Z global:
2024-04-03T01:44:04.121566304Z   evaluation_interval: 15s
2024-04-03T01:44:04.121570790Z   scrape_interval: 15s
2024-04-03T01:44:04.121574832Z rule_files:
2024-04-03T01:44:04.121578887Z - osm_metric_rules.yml
2024-04-03T01:44:04.121582853Z - osm_alert_rules.yml
2024-04-03T01:44:04.121586784Z scrape_configs:
2024-04-03T01:44:04.121591246Z - job_name: mon_exporter
2024-04-03T01:44:04.121595286Z   static_configs:
2024-04-03T01:44:04.121599340Z   - targets:
2024-04-03T01:44:04.121603425Z     - mon:8000
2024-04-03T01:44:04.121607381Z - honor_labels: true
2024-04-03T01:44:04.121611339Z   job_name: pushgateway
2024-04-03T01:44:04.121628741Z   scrape_interval: 30s
2024-04-03T01:44:04.121632873Z   static_configs:
2024-04-03T01:44:04.121636871Z   - targets:
2024-04-03T01:44:04.121644174Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:44:04.121648368Z 
2024-04-03T01:44:04.127678134Z Alerts read: 6
2024-04-03T01:44:04.129548588Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:44:04.136333938Z groups:
2024-04-03T01:44:04.136392194Z - name: osm_alert_rules
2024-04-03T01:44:04.136400623Z   rules:
2024-04-03T01:44:04.136409052Z   - alert: vdu_down
2024-04-03T01:44:04.136417284Z     annotations:
2024-04-03T01:44:04.136426555Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:44:04.136467632Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:44:04.136475879Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:44:04.136484502Z     expr: vm_status_extended != 1
2024-04-03T01:44:04.136493010Z     for: 3m
2024-04-03T01:44:04.136500993Z - name: _osm_alert_rules_20240403014404_
2024-04-03T01:44:04.136508825Z   rules:
2024-04-03T01:44:04.136556417Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:44:04.136565135Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136573957Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136581699Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:04.136589322Z     for: 1m
2024-04-03T01:44:04.136596974Z     labels:
2024-04-03T01:44:04.136630380Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.136638407Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.136646953Z       vnf_member_index: vnf
2024-04-03T01:44:04.136654510Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:44:04.136662251Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136670010Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136728533Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:04.136736660Z     for: 1m
2024-04-03T01:44:04.136744275Z     labels:
2024-04-03T01:44:04.136752040Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.136759629Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.136767207Z       vnf_member_index: vnf
2024-04-03T01:44:04.136796522Z   - alert: scalein_7fefd11f-bbce-4b79-961a-7dd5ca411880
2024-04-03T01:44:04.136804762Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136812626Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136820191Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:04.136827768Z     for: 1m
2024-04-03T01:44:04.136835447Z     labels:
2024-04-03T01:44:04.136843022Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.136850549Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.136887122Z       vnf_member_index: vnf
2024-04-03T01:44:04.136894839Z   - alert: scaleout_3ddff1ef-8bd4-4425-ac5a-ee18c921f211
2024-04-03T01:44:04.136902467Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136910139Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.136917751Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:04.136925368Z     for: 1m
2024-04-03T01:44:04.136932964Z     labels:
2024-04-03T01:44:04.136940491Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.137072076Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.137079976Z       vnf_member_index: vnf
2024-04-03T01:44:04.137087942Z   - alert: scalein_2664e90b-0c32-4aa2-88f7-8c41650bb41e
2024-04-03T01:44:04.137095473Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.137103122Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.137110861Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:04.137118498Z     for: 1m
2024-04-03T01:44:04.137150526Z     labels:
2024-04-03T01:44:04.137158725Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.137166408Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.137174187Z       vnf_member_index: vnf
2024-04-03T01:44:04.137181732Z   - alert: scaleout_66d78076-2600-47b9-b617-a3f4fc102bbf
2024-04-03T01:44:04.137189245Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.137197007Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:04.137204556Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:04.137236395Z     for: 1m
2024-04-03T01:44:04.137244286Z     labels:
2024-04-03T01:44:04.137251912Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:04.137259463Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:04.137267143Z       vnf_member_index: vnf
2024-04-03T01:44:04.137274701Z 
2024-04-03T01:44:04.142495306Z New config written, updating prometheus
2024-04-03T01:44:09.165265008Z Prometheus config update successful
2024-04-03T01:44:09.165294514Z 
2024-04-03T01:44:09.165300180Z Changes detected, updating prometheus config
2024-04-03T01:44:09.169363140Z Jobs detected: 0
2024-04-03T01:44:09.176545603Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:44:09.178396865Z alerting:
2024-04-03T01:44:09.178408022Z   alertmanagers:
2024-04-03T01:44:09.178411601Z   - static_configs:
2024-04-03T01:44:09.178414833Z     - targets:
2024-04-03T01:44:09.178418258Z       - alertmanager:9093
2024-04-03T01:44:09.178421282Z global:
2024-04-03T01:44:09.178424330Z   evaluation_interval: 15s
2024-04-03T01:44:09.178428095Z   scrape_interval: 15s
2024-04-03T01:44:09.178431104Z rule_files:
2024-04-03T01:44:09.178435534Z - osm_metric_rules.yml
2024-04-03T01:44:09.178440236Z - osm_alert_rules.yml
2024-04-03T01:44:09.178448428Z scrape_configs:
2024-04-03T01:44:09.178452604Z - job_name: mon_exporter
2024-04-03T01:44:09.178456734Z   static_configs:
2024-04-03T01:44:09.178461370Z   - targets:
2024-04-03T01:44:09.178465985Z     - mon:8000
2024-04-03T01:44:09.178470891Z - honor_labels: true
2024-04-03T01:44:09.178474249Z   job_name: pushgateway
2024-04-03T01:44:09.178477146Z   scrape_interval: 30s
2024-04-03T01:44:09.178480172Z   static_configs:
2024-04-03T01:44:09.178483056Z   - targets:
2024-04-03T01:44:09.178486473Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:44:09.178489425Z 
2024-04-03T01:44:09.192632022Z Alerts read: 6
2024-04-03T01:44:09.195464883Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:44:09.201877810Z groups:
2024-04-03T01:44:09.201891593Z - name: osm_alert_rules
2024-04-03T01:44:09.201895336Z   rules:
2024-04-03T01:44:09.201898731Z   - alert: vdu_down
2024-04-03T01:44:09.201905447Z     annotations:
2024-04-03T01:44:09.201909345Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:44:09.201913137Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:44:09.201916330Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:44:09.201920060Z     expr: vm_status_extended != 1
2024-04-03T01:44:09.201923184Z     for: 3m
2024-04-03T01:44:09.201926370Z - name: _osm_alert_rules_20240403014409_
2024-04-03T01:44:09.201929498Z   rules:
2024-04-03T01:44:09.201933389Z   - alert: scalein_8836c2cc-2342-45f3-9ec5-16dfa23defd2
2024-04-03T01:44:09.201936513Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.201940653Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.201943828Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:09.201946892Z     for: 1m
2024-04-03T01:44:09.201949899Z     labels:
2024-04-03T01:44:09.201952915Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.201955967Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.201959466Z       vnf_member_index: vnf
2024-04-03T01:44:09.201962578Z   - alert: scaleout_7088f7da-675e-4f9e-98b0-f5c0d782caf6
2024-04-03T01:44:09.201965711Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.201968749Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.201971775Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:09.201994799Z     for: 1m
2024-04-03T01:44:09.201997918Z     labels:
2024-04-03T01:44:09.202001247Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.202004304Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.202007348Z       vnf_member_index: vnf
2024-04-03T01:44:09.202010540Z   - alert: scalein_7fefd11f-bbce-4b79-961a-7dd5ca411880
2024-04-03T01:44:09.202013777Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202016971Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202020130Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:09.202023193Z     for: 1m
2024-04-03T01:44:09.202026175Z     labels:
2024-04-03T01:44:09.202029257Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.202032354Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.202036524Z       vnf_member_index: vnf
2024-04-03T01:44:09.202039654Z   - alert: scaleout_3ddff1ef-8bd4-4425-ac5a-ee18c921f211
2024-04-03T01:44:09.202054403Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202057480Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202060417Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:09.202063283Z     for: 1m
2024-04-03T01:44:09.202066234Z     labels:
2024-04-03T01:44:09.202069117Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.202072018Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.202074989Z       vnf_member_index: vnf
2024-04-03T01:44:09.202077886Z   - alert: scalein_2664e90b-0c32-4aa2-88f7-8c41650bb41e
2024-04-03T01:44:09.202080816Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202083919Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 3) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202086955Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T01:44:09.202089823Z     for: 1m
2024-04-03T01:44:09.202092709Z     labels:
2024-04-03T01:44:09.202095599Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.202098458Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.202101410Z       vnf_member_index: vnf
2024-04-03T01:44:09.202104366Z   - alert: scaleout_66d78076-2600-47b9-b617-a3f4fc102bbf
2024-04-03T01:44:09.202107576Z     expr: (count (osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202110524Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 5) and (avg(osm_cpu_utilization{ns_id="ee015990-40b5-47d0-8498-7d636cd95ffd",
2024-04-03T01:44:09.202113401Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T01:44:09.202116256Z     for: 1m
2024-04-03T01:44:09.202119172Z     labels:
2024-04-03T01:44:09.202122330Z       ns_id: ee015990-40b5-47d0-8498-7d636cd95ffd
2024-04-03T01:44:09.202125256Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T01:44:09.202128187Z       vnf_member_index: vnf
2024-04-03T01:44:09.202130949Z 
2024-04-03T01:44:09.208882626Z New config written, updating prometheus
2024-04-03T01:44:14.244051135Z Prometheus config update successful
2024-04-03T01:44:14.244101192Z 
2024-04-03T01:45:31.378054075Z Changes detected, updating prometheus config
2024-04-03T01:45:31.381958406Z Jobs detected: 0
2024-04-03T01:45:31.397760941Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:31.401687712Z alerting:
2024-04-03T01:45:31.401712541Z   alertmanagers:
2024-04-03T01:45:31.401719363Z   - static_configs:
2024-04-03T01:45:31.401723753Z     - targets:
2024-04-03T01:45:31.401742264Z       - alertmanager:9093
2024-04-03T01:45:31.401748020Z global:
2024-04-03T01:45:31.401752963Z   evaluation_interval: 15s
2024-04-03T01:45:31.401757337Z   scrape_interval: 15s
2024-04-03T01:45:31.401761815Z rule_files:
2024-04-03T01:45:31.401766311Z - osm_metric_rules.yml
2024-04-03T01:45:31.401770635Z - osm_alert_rules.yml
2024-04-03T01:45:31.401775277Z scrape_configs:
2024-04-03T01:45:31.401779741Z - job_name: mon_exporter
2024-04-03T01:45:31.401784363Z   static_configs:
2024-04-03T01:45:31.401789012Z   - targets:
2024-04-03T01:45:31.401793584Z     - mon:8000
2024-04-03T01:45:31.401798591Z - honor_labels: true
2024-04-03T01:45:31.401803378Z   job_name: pushgateway
2024-04-03T01:45:31.401818384Z   scrape_interval: 30s
2024-04-03T01:45:31.401824040Z   static_configs:
2024-04-03T01:45:31.401829132Z   - targets:
2024-04-03T01:45:31.401834657Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:31.401839650Z 
2024-04-03T01:45:31.409440438Z Alerts read: 0
2024-04-03T01:45:31.412139078Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:31.413408773Z groups:
2024-04-03T01:45:31.413425140Z - name: osm_alert_rules
2024-04-03T01:45:31.413437899Z   rules:
2024-04-03T01:45:31.413441208Z   - alert: vdu_down
2024-04-03T01:45:31.413444075Z     annotations:
2024-04-03T01:45:31.413448144Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:31.413452123Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:31.413460809Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:31.413465066Z     expr: vm_status_extended != 1
2024-04-03T01:45:31.413469857Z     for: 3m
2024-04-03T01:45:31.413474680Z 
2024-04-03T01:45:31.414744512Z New config written, updating prometheus
2024-04-03T01:45:36.445319329Z Prometheus config update successful
2024-04-03T01:45:36.445376120Z 
2024-04-03T01:45:36.445381565Z Changes detected, updating prometheus config
2024-04-03T01:45:36.446451685Z Jobs detected: 0
2024-04-03T01:45:36.450999594Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:36.452700625Z alerting:
2024-04-03T01:45:36.452709199Z   alertmanagers:
2024-04-03T01:45:36.452712925Z   - static_configs:
2024-04-03T01:45:36.452715892Z     - targets:
2024-04-03T01:45:36.452719772Z       - alertmanager:9093
2024-04-03T01:45:36.452722863Z global:
2024-04-03T01:45:36.452726394Z   evaluation_interval: 15s
2024-04-03T01:45:36.452729484Z   scrape_interval: 15s
2024-04-03T01:45:36.452732271Z rule_files:
2024-04-03T01:45:36.452735065Z - osm_metric_rules.yml
2024-04-03T01:45:36.452737769Z - osm_alert_rules.yml
2024-04-03T01:45:36.452741479Z scrape_configs:
2024-04-03T01:45:36.452744256Z - job_name: mon_exporter
2024-04-03T01:45:36.452746969Z   static_configs:
2024-04-03T01:45:36.452750173Z   - targets:
2024-04-03T01:45:36.452753368Z     - mon:8000
2024-04-03T01:45:36.452756356Z - honor_labels: true
2024-04-03T01:45:36.452759157Z   job_name: pushgateway
2024-04-03T01:45:36.452776653Z   scrape_interval: 30s
2024-04-03T01:45:36.452779692Z   static_configs:
2024-04-03T01:45:36.452782429Z   - targets:
2024-04-03T01:45:36.452786150Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:36.452789213Z 
2024-04-03T01:45:36.455547749Z Alerts read: 0
2024-04-03T01:45:36.457324569Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:36.458324726Z groups:
2024-04-03T01:45:36.458331810Z - name: osm_alert_rules
2024-04-03T01:45:36.458334966Z   rules:
2024-04-03T01:45:36.458337937Z   - alert: vdu_down
2024-04-03T01:45:36.458340999Z     annotations:
2024-04-03T01:45:36.458344989Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:36.458348611Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:36.458351702Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:36.458355071Z     expr: vm_status_extended != 1
2024-04-03T01:45:36.458357961Z     for: 3m
2024-04-03T01:45:36.458360683Z 
2024-04-03T01:45:36.459515100Z New config written, updating prometheus
2024-04-03T01:45:41.490899653Z Prometheus config update successful
2024-04-03T01:45:41.490934170Z 
2024-04-03T01:45:41.491010903Z Changes detected, updating prometheus config
2024-04-03T01:45:41.492229481Z Jobs detected: 0
2024-04-03T01:45:41.498940030Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:41.500850360Z alerting:
2024-04-03T01:45:41.500860053Z   alertmanagers:
2024-04-03T01:45:41.500864829Z   - static_configs:
2024-04-03T01:45:41.500868082Z     - targets:
2024-04-03T01:45:41.500872166Z       - alertmanager:9093
2024-04-03T01:45:41.500875311Z global:
2024-04-03T01:45:41.500878370Z   evaluation_interval: 15s
2024-04-03T01:45:41.500881436Z   scrape_interval: 15s
2024-04-03T01:45:41.500884656Z rule_files:
2024-04-03T01:45:41.500887767Z - osm_metric_rules.yml
2024-04-03T01:45:41.500890935Z - osm_alert_rules.yml
2024-04-03T01:45:41.500894021Z scrape_configs:
2024-04-03T01:45:41.500897121Z - job_name: mon_exporter
2024-04-03T01:45:41.500900253Z   static_configs:
2024-04-03T01:45:41.500903246Z   - targets:
2024-04-03T01:45:41.500906883Z     - mon:8000
2024-04-03T01:45:41.500909831Z - honor_labels: true
2024-04-03T01:45:41.500912800Z   job_name: pushgateway
2024-04-03T01:45:41.500916061Z   scrape_interval: 30s
2024-04-03T01:45:41.500919049Z   static_configs:
2024-04-03T01:45:41.500922162Z   - targets:
2024-04-03T01:45:41.500925891Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:41.500928856Z 
2024-04-03T01:45:41.504090484Z Alerts read: 0
2024-04-03T01:45:41.505971099Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:41.507254895Z groups:
2024-04-03T01:45:41.507265779Z - name: osm_alert_rules
2024-04-03T01:45:41.507269142Z   rules:
2024-04-03T01:45:41.507272394Z   - alert: vdu_down
2024-04-03T01:45:41.507275534Z     annotations:
2024-04-03T01:45:41.507279556Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:41.507283578Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:41.507287072Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:41.507290872Z     expr: vm_status_extended != 1
2024-04-03T01:45:41.507293862Z     for: 3m
2024-04-03T01:45:41.507296634Z 
2024-04-03T01:45:41.508349869Z New config written, updating prometheus
2024-04-03T01:45:46.544310946Z Prometheus config update successful
2024-04-03T01:45:46.544376050Z 
2024-04-03T01:45:46.544385583Z Changes detected, updating prometheus config
2024-04-03T01:45:46.545561572Z Jobs detected: 0
2024-04-03T01:45:46.551679834Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:46.553707186Z alerting:
2024-04-03T01:45:46.553723051Z   alertmanagers:
2024-04-03T01:45:46.553729382Z   - static_configs:
2024-04-03T01:45:46.553734460Z     - targets:
2024-04-03T01:45:46.553740449Z       - alertmanager:9093
2024-04-03T01:45:46.553745690Z global:
2024-04-03T01:45:46.553750699Z   evaluation_interval: 15s
2024-04-03T01:45:46.553755616Z   scrape_interval: 15s
2024-04-03T01:45:46.553760705Z rule_files:
2024-04-03T01:45:46.553781107Z - osm_metric_rules.yml
2024-04-03T01:45:46.553786643Z - osm_alert_rules.yml
2024-04-03T01:45:46.553791533Z scrape_configs:
2024-04-03T01:45:46.553796382Z - job_name: mon_exporter
2024-04-03T01:45:46.553801961Z   static_configs:
2024-04-03T01:45:46.553806999Z   - targets:
2024-04-03T01:45:46.553811979Z     - mon:8000
2024-04-03T01:45:46.553816910Z - honor_labels: true
2024-04-03T01:45:46.553821897Z   job_name: pushgateway
2024-04-03T01:45:46.553826834Z   scrape_interval: 30s
2024-04-03T01:45:46.553831857Z   static_configs:
2024-04-03T01:45:46.553837456Z   - targets:
2024-04-03T01:45:46.553842944Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:46.553847800Z 
2024-04-03T01:45:46.556772773Z Alerts read: 0
2024-04-03T01:45:46.558514952Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:46.559621076Z groups:
2024-04-03T01:45:46.559637605Z - name: osm_alert_rules
2024-04-03T01:45:46.559643144Z   rules:
2024-04-03T01:45:46.559648176Z   - alert: vdu_down
2024-04-03T01:45:46.559653071Z     annotations:
2024-04-03T01:45:46.559658953Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:46.559664489Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:46.559669301Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:46.559674460Z     expr: vm_status_extended != 1
2024-04-03T01:45:46.559690125Z     for: 3m
2024-04-03T01:45:46.559695174Z 
2024-04-03T01:45:46.560491296Z New config written, updating prometheus
2024-04-03T01:45:51.582399269Z Prometheus config update successful
2024-04-03T01:45:51.582435335Z 
2024-04-03T01:45:51.582439177Z Changes detected, updating prometheus config
2024-04-03T01:45:51.583543194Z Jobs detected: 0
2024-04-03T01:45:51.588303568Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:51.590110827Z alerting:
2024-04-03T01:45:51.590136142Z   alertmanagers:
2024-04-03T01:45:51.590141364Z   - static_configs:
2024-04-03T01:45:51.590145520Z     - targets:
2024-04-03T01:45:51.590150351Z       - alertmanager:9093
2024-04-03T01:45:51.590154400Z global:
2024-04-03T01:45:51.590158518Z   evaluation_interval: 15s
2024-04-03T01:45:51.590163091Z   scrape_interval: 15s
2024-04-03T01:45:51.590167176Z rule_files:
2024-04-03T01:45:51.590171337Z - osm_metric_rules.yml
2024-04-03T01:45:51.590175343Z - osm_alert_rules.yml
2024-04-03T01:45:51.590179277Z scrape_configs:
2024-04-03T01:45:51.590183332Z - job_name: mon_exporter
2024-04-03T01:45:51.590187352Z   static_configs:
2024-04-03T01:45:51.590191360Z   - targets:
2024-04-03T01:45:51.590195576Z     - mon:8000
2024-04-03T01:45:51.590205406Z - honor_labels: true
2024-04-03T01:45:51.590209503Z   job_name: pushgateway
2024-04-03T01:45:51.590226964Z   scrape_interval: 30s
2024-04-03T01:45:51.590231069Z   static_configs:
2024-04-03T01:45:51.590235145Z   - targets:
2024-04-03T01:45:51.590239771Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:51.590243726Z 
2024-04-03T01:45:51.593756410Z Alerts read: 0
2024-04-03T01:45:51.595558724Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:51.596626043Z groups:
2024-04-03T01:45:51.596648585Z - name: osm_alert_rules
2024-04-03T01:45:51.596653071Z   rules:
2024-04-03T01:45:51.596657501Z   - alert: vdu_down
2024-04-03T01:45:51.596661546Z     annotations:
2024-04-03T01:45:51.596666536Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:51.596671903Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:51.596676042Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:51.596680760Z     expr: vm_status_extended != 1
2024-04-03T01:45:51.596684942Z     for: 3m
2024-04-03T01:45:51.596688760Z 
2024-04-03T01:45:51.597721836Z New config written, updating prometheus
2024-04-03T01:45:56.620925490Z Prometheus config update successful
2024-04-03T01:45:56.627190400Z 
2024-04-03T01:45:56.627214424Z Changes detected, updating prometheus config
2024-04-03T01:45:56.630681635Z Jobs detected: 0
2024-04-03T01:45:56.630698070Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T01:45:56.631055359Z alerting:
2024-04-03T01:45:56.631064082Z   alertmanagers:
2024-04-03T01:45:56.631068020Z   - static_configs:
2024-04-03T01:45:56.631071079Z     - targets:
2024-04-03T01:45:56.631074571Z       - alertmanager:9093
2024-04-03T01:45:56.631077565Z global:
2024-04-03T01:45:56.631080787Z   evaluation_interval: 15s
2024-04-03T01:45:56.631083833Z   scrape_interval: 15s
2024-04-03T01:45:56.631086863Z rule_files:
2024-04-03T01:45:56.631090087Z - osm_metric_rules.yml
2024-04-03T01:45:56.631093062Z - osm_alert_rules.yml
2024-04-03T01:45:56.631096040Z scrape_configs:
2024-04-03T01:45:56.631099015Z - job_name: mon_exporter
2024-04-03T01:45:56.631102455Z   static_configs:
2024-04-03T01:45:56.631105499Z   - targets:
2024-04-03T01:45:56.631108431Z     - mon:8000
2024-04-03T01:45:56.631111397Z - honor_labels: true
2024-04-03T01:45:56.631114381Z   job_name: pushgateway
2024-04-03T01:45:56.631117322Z   scrape_interval: 30s
2024-04-03T01:45:56.631120318Z   static_configs:
2024-04-03T01:45:56.631123340Z   - targets:
2024-04-03T01:45:56.631126859Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T01:45:56.631129796Z 
2024-04-03T01:45:56.635975957Z Alerts read: 0
2024-04-03T01:45:56.639712355Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T01:45:56.641293929Z groups:
2024-04-03T01:45:56.641305330Z - name: osm_alert_rules
2024-04-03T01:45:56.641311538Z   rules:
2024-04-03T01:45:56.641314861Z   - alert: vdu_down
2024-04-03T01:45:56.641318055Z     annotations:
2024-04-03T01:45:56.641321418Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T01:45:56.641324836Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T01:45:56.641328654Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T01:45:56.641333249Z     expr: vm_status_extended != 1
2024-04-03T01:45:56.641337739Z     for: 3m
2024-04-03T01:45:56.641350837Z 
2024-04-03T01:45:56.642355132Z New config written, updating prometheus
2024-04-03T01:46:01.677793826Z Prometheus config update successful
2024-04-03T01:46:01.677818486Z 
2024-04-03T02:46:05.524361191Z Changes detected, updating prometheus config
2024-04-03T02:46:05.527781905Z Jobs detected: 0
2024-04-03T02:46:05.533526384Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T02:46:05.535556936Z alerting:
2024-04-03T02:46:05.535567099Z   alertmanagers:
2024-04-03T02:46:05.535571606Z   - static_configs:
2024-04-03T02:46:05.535574352Z     - targets:
2024-04-03T02:46:05.535577671Z       - alertmanager:9093
2024-04-03T02:46:05.535581338Z global:
2024-04-03T02:46:05.535584436Z   evaluation_interval: 15s
2024-04-03T02:46:05.535587106Z   scrape_interval: 15s
2024-04-03T02:46:05.535589708Z rule_files:
2024-04-03T02:46:05.535592397Z - osm_metric_rules.yml
2024-04-03T02:46:05.535595151Z - osm_alert_rules.yml
2024-04-03T02:46:05.535597908Z scrape_configs:
2024-04-03T02:46:05.535600588Z - job_name: mon_exporter
2024-04-03T02:46:05.535603198Z   static_configs:
2024-04-03T02:46:05.535640151Z   - targets:
2024-04-03T02:46:05.535643221Z     - mon:8000
2024-04-03T02:46:05.535645950Z - honor_labels: true
2024-04-03T02:46:05.535648564Z   job_name: pushgateway
2024-04-03T02:46:05.535651562Z   scrape_interval: 30s
2024-04-03T02:46:05.535654290Z   static_configs:
2024-04-03T02:46:05.535656770Z   - targets:
2024-04-03T02:46:05.535660526Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T02:46:05.535663177Z 
2024-04-03T02:46:05.539450369Z Alerts read: 0
2024-04-03T02:46:05.541612220Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T02:46:05.542997225Z groups:
2024-04-03T02:46:05.543009737Z - name: osm_alert_rules
2024-04-03T02:46:05.543012723Z   rules:
2024-04-03T02:46:05.543015858Z   - alert: vdu_down
2024-04-03T02:46:05.543019219Z     annotations:
2024-04-03T02:46:05.543023176Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T02:46:05.543027297Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T02:46:05.543030243Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T02:46:05.543034153Z     expr: vm_status_extended != 1
2024-04-03T02:46:05.543037017Z     for: 3m
2024-04-03T02:46:05.543039635Z 
2024-04-03T02:46:05.544118099Z New config written, updating prometheus
2024-04-03T02:46:10.579912010Z Prometheus config update successful
2024-04-03T02:46:10.579940296Z 
2024-04-03T02:53:24.269095272Z Changes detected, updating prometheus config
2024-04-03T02:53:24.272035794Z Jobs detected: 0
2024-04-03T02:53:24.281661023Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T02:53:24.284575781Z alerting:
2024-04-03T02:53:24.284600058Z   alertmanagers:
2024-04-03T02:53:24.284608428Z   - static_configs:
2024-04-03T02:53:24.284622469Z     - targets:
2024-04-03T02:53:24.284630482Z       - alertmanager:9093
2024-04-03T02:53:24.284637554Z global:
2024-04-03T02:53:24.284644681Z   evaluation_interval: 15s
2024-04-03T02:53:24.284652141Z   scrape_interval: 15s
2024-04-03T02:53:24.284659386Z rule_files:
2024-04-03T02:53:24.284666424Z - osm_metric_rules.yml
2024-04-03T02:53:24.284673398Z - osm_alert_rules.yml
2024-04-03T02:53:24.284680350Z scrape_configs:
2024-04-03T02:53:24.284687549Z - job_name: mon_exporter
2024-04-03T02:53:24.284695001Z   static_configs:
2024-04-03T02:53:24.284702014Z   - targets:
2024-04-03T02:53:24.284709312Z     - mon:8000
2024-04-03T02:53:24.284733090Z - honor_labels: true
2024-04-03T02:53:24.284740715Z   job_name: pushgateway
2024-04-03T02:53:24.284768256Z   scrape_interval: 30s
2024-04-03T02:53:24.284775549Z   static_configs:
2024-04-03T02:53:24.284782849Z   - targets:
2024-04-03T02:53:24.284790675Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T02:53:24.284797588Z 
2024-04-03T02:53:24.288008363Z Alerts read: 0
2024-04-03T02:53:24.290132457Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T02:53:24.291256397Z groups:
2024-04-03T02:53:24.291277300Z - name: osm_alert_rules
2024-04-03T02:53:24.291284486Z   rules:
2024-04-03T02:53:24.291291253Z   - alert: vdu_down
2024-04-03T02:53:24.291297622Z     annotations:
2024-04-03T02:53:24.291304745Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T02:53:24.291312139Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T02:53:24.291318326Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T02:53:24.291325530Z     expr: vm_status_extended != 1
2024-04-03T02:53:24.291331868Z     for: 3m
2024-04-03T02:53:24.291337893Z 
2024-04-03T02:53:24.292417271Z New config written, updating prometheus
2024-04-03T02:53:29.329560566Z Prometheus config update successful
2024-04-03T02:53:29.329589350Z 
2024-04-03T03:18:59.181461219Z Changes detected, updating prometheus config
2024-04-03T03:18:59.183432392Z Jobs detected: 0
2024-04-03T03:18:59.188477881Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:18:59.190476132Z alerting:
2024-04-03T03:18:59.190486617Z   alertmanagers:
2024-04-03T03:18:59.190490360Z   - static_configs:
2024-04-03T03:18:59.190493507Z     - targets:
2024-04-03T03:18:59.190497263Z       - alertmanager:9093
2024-04-03T03:18:59.190500277Z global:
2024-04-03T03:18:59.190503347Z   evaluation_interval: 15s
2024-04-03T03:18:59.190506368Z   scrape_interval: 15s
2024-04-03T03:18:59.190509380Z rule_files:
2024-04-03T03:18:59.190512340Z - osm_metric_rules.yml
2024-04-03T03:18:59.190515340Z - osm_alert_rules.yml
2024-04-03T03:18:59.190535269Z scrape_configs:
2024-04-03T03:18:59.190539431Z - job_name: mon_exporter
2024-04-03T03:18:59.190542455Z   static_configs:
2024-04-03T03:18:59.190545439Z   - targets:
2024-04-03T03:18:59.190548796Z     - mon:8000
2024-04-03T03:18:59.190551706Z - honor_labels: true
2024-04-03T03:18:59.190554640Z   job_name: pushgateway
2024-04-03T03:18:59.190557980Z   scrape_interval: 30s
2024-04-03T03:18:59.190560948Z   static_configs:
2024-04-03T03:18:59.190564027Z   - targets:
2024-04-03T03:18:59.190567491Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:18:59.190570401Z 
2024-04-03T03:18:59.194151854Z Alerts read: 2
2024-04-03T03:18:59.196132944Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:18:59.201775252Z groups:
2024-04-03T03:18:59.201790590Z - name: osm_alert_rules
2024-04-03T03:18:59.201796402Z   rules:
2024-04-03T03:18:59.201801137Z   - alert: vdu_down
2024-04-03T03:18:59.201805802Z     annotations:
2024-04-03T03:18:59.201843803Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:18:59.201849940Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:18:59.201854922Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:18:59.201860391Z     expr: vm_status_extended != 1
2024-04-03T03:18:59.201864693Z     for: 3m
2024-04-03T03:18:59.201869329Z - name: _osm_alert_rules_20240403031859_
2024-04-03T03:18:59.201874077Z   rules:
2024-04-03T03:18:59.201906398Z   - alert: scalein_8edc8851-8dea-413b-9d9c-820821dfc4e3
2024-04-03T03:18:59.201911259Z     expr: (count (osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:18:59.201916363Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:18:59.201920934Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T03:18:59.201925595Z     for: 1m
2024-04-03T03:18:59.201930199Z     labels:
2024-04-03T03:18:59.201934734Z       ns_id: 8241544d-831d-4177-86d7-52c8b5e2f35b
2024-04-03T03:18:59.201939390Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:18:59.201944342Z       vnf_member_index: vnf
2024-04-03T03:18:59.201948796Z   - alert: scaleout_e75c1764-8cec-46a3-883c-ab45b042cd55
2024-04-03T03:18:59.201953060Z     expr: (count (osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:18:59.201957348Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:18:59.201961962Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T03:18:59.201966433Z     for: 1m
2024-04-03T03:18:59.201970867Z     labels:
2024-04-03T03:18:59.201975555Z       ns_id: 8241544d-831d-4177-86d7-52c8b5e2f35b
2024-04-03T03:18:59.201980078Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:18:59.201984560Z       vnf_member_index: vnf
2024-04-03T03:18:59.201988659Z 
2024-04-03T03:18:59.208312066Z New config written, updating prometheus
2024-04-03T03:19:04.264651178Z Prometheus config update successful
2024-04-03T03:19:04.264669328Z 
2024-04-03T03:19:04.267457071Z Changes detected, updating prometheus config
2024-04-03T03:19:04.269184594Z Jobs detected: 0
2024-04-03T03:19:04.282783247Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:19:04.286747338Z alerting:
2024-04-03T03:19:04.286823474Z   alertmanagers:
2024-04-03T03:19:04.286831814Z   - static_configs:
2024-04-03T03:19:04.286838375Z     - targets:
2024-04-03T03:19:04.286873386Z       - alertmanager:9093
2024-04-03T03:19:04.286881283Z global:
2024-04-03T03:19:04.286887042Z   evaluation_interval: 15s
2024-04-03T03:19:04.286893043Z   scrape_interval: 15s
2024-04-03T03:19:04.286898688Z rule_files:
2024-04-03T03:19:04.286904285Z - osm_metric_rules.yml
2024-04-03T03:19:04.286910073Z - osm_alert_rules.yml
2024-04-03T03:19:04.286915793Z scrape_configs:
2024-04-03T03:19:04.286921497Z - job_name: mon_exporter
2024-04-03T03:19:04.286927184Z   static_configs:
2024-04-03T03:19:04.286951972Z   - targets:
2024-04-03T03:19:04.286958047Z     - mon:8000
2024-04-03T03:19:04.286963884Z - honor_labels: true
2024-04-03T03:19:04.286970439Z   job_name: pushgateway
2024-04-03T03:19:04.286976224Z   scrape_interval: 30s
2024-04-03T03:19:04.286981958Z   static_configs:
2024-04-03T03:19:04.286987618Z   - targets:
2024-04-03T03:19:04.286996637Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:19:04.287002211Z 
2024-04-03T03:19:04.293673364Z Alerts read: 2
2024-04-03T03:19:04.297069466Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:19:04.304247441Z groups:
2024-04-03T03:19:04.304291987Z - name: osm_alert_rules
2024-04-03T03:19:04.304295822Z   rules:
2024-04-03T03:19:04.304298957Z   - alert: vdu_down
2024-04-03T03:19:04.304302052Z     annotations:
2024-04-03T03:19:04.304318729Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:19:04.304345080Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:19:04.304348231Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:19:04.304352249Z     expr: vm_status_extended != 1
2024-04-03T03:19:04.304368187Z     for: 3m
2024-04-03T03:19:04.304371957Z - name: _osm_alert_rules_20240403031904_
2024-04-03T03:19:04.304375038Z   rules:
2024-04-03T03:19:04.304378614Z   - alert: scalein_8edc8851-8dea-413b-9d9c-820821dfc4e3
2024-04-03T03:19:04.304381756Z     expr: (count (osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:19:04.304385784Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:19:04.304388821Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T03:19:04.304391838Z     for: 1m
2024-04-03T03:19:04.304394823Z     labels:
2024-04-03T03:19:04.304397917Z       ns_id: 8241544d-831d-4177-86d7-52c8b5e2f35b
2024-04-03T03:19:04.304400901Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:19:04.304404295Z       vnf_member_index: vnf
2024-04-03T03:19:04.304407335Z   - alert: scaleout_e75c1764-8cec-46a3-883c-ab45b042cd55
2024-04-03T03:19:04.304410431Z     expr: (count (osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:19:04.304413738Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="8241544d-831d-4177-86d7-52c8b5e2f35b",
2024-04-03T03:19:04.304416716Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T03:19:04.304419668Z     for: 1m
2024-04-03T03:19:04.304422525Z     labels:
2024-04-03T03:19:04.304425469Z       ns_id: 8241544d-831d-4177-86d7-52c8b5e2f35b
2024-04-03T03:19:04.304428543Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:19:04.304431434Z       vnf_member_index: vnf
2024-04-03T03:19:04.304434199Z 
2024-04-03T03:19:04.308346567Z New config written, updating prometheus
2024-04-03T03:19:09.350769267Z Prometheus config update successful
2024-04-03T03:19:09.350803405Z 
2024-04-03T03:26:24.146985663Z Changes detected, updating prometheus config
2024-04-03T03:26:24.148759457Z Jobs detected: 0
2024-04-03T03:26:24.154212480Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:26:24.157277826Z alerting:
2024-04-03T03:26:24.157289585Z   alertmanagers:
2024-04-03T03:26:24.157293375Z   - static_configs:
2024-04-03T03:26:24.157296686Z     - targets:
2024-04-03T03:26:24.157303614Z       - alertmanager:9093
2024-04-03T03:26:24.157306828Z global:
2024-04-03T03:26:24.157309935Z   evaluation_interval: 15s
2024-04-03T03:26:24.157313209Z   scrape_interval: 15s
2024-04-03T03:26:24.157316531Z rule_files:
2024-04-03T03:26:24.157319685Z - osm_metric_rules.yml
2024-04-03T03:26:24.157322685Z - osm_alert_rules.yml
2024-04-03T03:26:24.157325758Z scrape_configs:
2024-04-03T03:26:24.157328936Z - job_name: mon_exporter
2024-04-03T03:26:24.157332399Z   static_configs:
2024-04-03T03:26:24.157335477Z   - targets:
2024-04-03T03:26:24.157339049Z     - mon:8000
2024-04-03T03:26:24.157342161Z - honor_labels: true
2024-04-03T03:26:24.157345207Z   job_name: pushgateway
2024-04-03T03:26:24.157348189Z   scrape_interval: 30s
2024-04-03T03:26:24.157351577Z   static_configs:
2024-04-03T03:26:24.157354566Z   - targets:
2024-04-03T03:26:24.157358403Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:26:24.157446447Z 
2024-04-03T03:26:24.160377421Z Alerts read: 0
2024-04-03T03:26:24.162510723Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:26:24.163689660Z groups:
2024-04-03T03:26:24.163700508Z - name: osm_alert_rules
2024-04-03T03:26:24.163703814Z   rules:
2024-04-03T03:26:24.163706817Z   - alert: vdu_down
2024-04-03T03:26:24.163709772Z     annotations:
2024-04-03T03:26:24.163713765Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:26:24.163720021Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:26:24.163723106Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:26:24.163726717Z     expr: vm_status_extended != 1
2024-04-03T03:26:24.163730350Z     for: 3m
2024-04-03T03:26:24.163900416Z 
2024-04-03T03:26:24.165135146Z New config written, updating prometheus
2024-04-03T03:26:29.193249123Z Prometheus config update successful
2024-04-03T03:26:29.193347550Z 
2024-04-03T03:26:29.193500454Z Changes detected, updating prometheus config
2024-04-03T03:26:29.194848896Z Jobs detected: 0
2024-04-03T03:26:29.200121857Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:26:29.202043672Z alerting:
2024-04-03T03:26:29.202054455Z   alertmanagers:
2024-04-03T03:26:29.202058342Z   - static_configs:
2024-04-03T03:26:29.202061701Z     - targets:
2024-04-03T03:26:29.202065266Z       - alertmanager:9093
2024-04-03T03:26:29.202068355Z global:
2024-04-03T03:26:29.202071531Z   evaluation_interval: 15s
2024-04-03T03:26:29.202074605Z   scrape_interval: 15s
2024-04-03T03:26:29.202077637Z rule_files:
2024-04-03T03:26:29.202080764Z - osm_metric_rules.yml
2024-04-03T03:26:29.202083769Z - osm_alert_rules.yml
2024-04-03T03:26:29.202086891Z scrape_configs:
2024-04-03T03:26:29.202090010Z - job_name: mon_exporter
2024-04-03T03:26:29.202092994Z   static_configs:
2024-04-03T03:26:29.202096070Z   - targets:
2024-04-03T03:26:29.202099107Z     - mon:8000
2024-04-03T03:26:29.202102156Z - honor_labels: true
2024-04-03T03:26:29.202105592Z   job_name: pushgateway
2024-04-03T03:26:29.202108650Z   scrape_interval: 30s
2024-04-03T03:26:29.202111788Z   static_configs:
2024-04-03T03:26:29.202114818Z   - targets:
2024-04-03T03:26:29.202118422Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:26:29.202416339Z 
2024-04-03T03:26:29.205930169Z Alerts read: 0
2024-04-03T03:26:29.208668856Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:26:29.209930742Z groups:
2024-04-03T03:26:29.209941502Z - name: osm_alert_rules
2024-04-03T03:26:29.209944822Z   rules:
2024-04-03T03:26:29.209948022Z   - alert: vdu_down
2024-04-03T03:26:29.209951841Z     annotations:
2024-04-03T03:26:29.209955674Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:26:29.209959455Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:26:29.209962548Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:26:29.209966183Z     expr: vm_status_extended != 1
2024-04-03T03:26:29.209969213Z     for: 3m
2024-04-03T03:26:29.210116573Z 
2024-04-03T03:26:29.211823934Z New config written, updating prometheus
2024-04-03T03:26:34.237653454Z Prometheus config update successful
2024-04-03T03:26:34.237742215Z 
2024-04-03T03:27:14.481163594Z Changes detected, updating prometheus config
2024-04-03T03:27:14.483077499Z Jobs detected: 0
2024-04-03T03:27:14.488003019Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:27:14.489740998Z alerting:
2024-04-03T03:27:14.489752640Z   alertmanagers:
2024-04-03T03:27:14.489756480Z   - static_configs:
2024-04-03T03:27:14.489759929Z     - targets:
2024-04-03T03:27:14.489763491Z       - alertmanager:9093
2024-04-03T03:27:14.489766440Z global:
2024-04-03T03:27:14.489769406Z   evaluation_interval: 15s
2024-04-03T03:27:14.489772327Z   scrape_interval: 15s
2024-04-03T03:27:14.489775289Z rule_files:
2024-04-03T03:27:14.489778271Z - osm_metric_rules.yml
2024-04-03T03:27:14.489781105Z - osm_alert_rules.yml
2024-04-03T03:27:14.489784137Z scrape_configs:
2024-04-03T03:27:14.489787056Z - job_name: mon_exporter
2024-04-03T03:27:14.489789967Z   static_configs:
2024-04-03T03:27:14.489792796Z   - targets:
2024-04-03T03:27:14.489795738Z     - mon:8000
2024-04-03T03:27:14.489798643Z - honor_labels: true
2024-04-03T03:27:14.489801543Z   job_name: pushgateway
2024-04-03T03:27:14.489804446Z   scrape_interval: 30s
2024-04-03T03:27:14.489807360Z   static_configs:
2024-04-03T03:27:14.489810635Z   - targets:
2024-04-03T03:27:14.489814754Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:27:14.489817631Z 
2024-04-03T03:27:14.493270212Z Alerts read: 2
2024-04-03T03:27:14.495138633Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:27:14.498155216Z groups:
2024-04-03T03:27:14.498167411Z - name: osm_alert_rules
2024-04-03T03:27:14.498170794Z   rules:
2024-04-03T03:27:14.498173929Z   - alert: vdu_down
2024-04-03T03:27:14.498177004Z     annotations:
2024-04-03T03:27:14.498180700Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:27:14.498201262Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:27:14.498205261Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:27:14.498208951Z     expr: vm_status_extended != 1
2024-04-03T03:27:14.498212020Z     for: 3m
2024-04-03T03:27:14.498215127Z - name: _osm_alert_rules_20240403032714_
2024-04-03T03:27:14.498218218Z   rules:
2024-04-03T03:27:14.498222379Z   - alert: scalein_a438e133-20bb-4d40-bc7c-8359c4006bf2
2024-04-03T03:27:14.498225557Z     expr: (count (osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:14.498235491Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:14.498238768Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T03:27:14.498242103Z     for: 1m
2024-04-03T03:27:14.498245075Z     labels:
2024-04-03T03:27:14.498248143Z       ns_id: aa5dac12-c347-43c8-b4ce-bfa740b9f1b3
2024-04-03T03:27:14.498251215Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:27:14.498254971Z       vnf_member_index: vnf
2024-04-03T03:27:14.498258353Z   - alert: scaleout_1ff16aa4-c0be-4101-b6ea-126902dfc530
2024-04-03T03:27:14.498261509Z     expr: (count (osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:14.498264718Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:14.498267746Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T03:27:14.498270764Z     for: 1m
2024-04-03T03:27:14.498273774Z     labels:
2024-04-03T03:27:14.498276844Z       ns_id: aa5dac12-c347-43c8-b4ce-bfa740b9f1b3
2024-04-03T03:27:14.498283625Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:27:14.498307404Z       vnf_member_index: vnf
2024-04-03T03:27:14.498310450Z 
2024-04-03T03:27:14.501061873Z New config written, updating prometheus
2024-04-03T03:27:19.524503334Z Prometheus config update successful
2024-04-03T03:27:19.524530120Z 
2024-04-03T03:27:19.531605890Z Changes detected, updating prometheus config
2024-04-03T03:27:19.532457259Z Jobs detected: 0
2024-04-03T03:27:19.538134781Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:27:19.541470305Z alerting:
2024-04-03T03:27:19.541484434Z   alertmanagers:
2024-04-03T03:27:19.541488451Z   - static_configs:
2024-04-03T03:27:19.541491489Z     - targets:
2024-04-03T03:27:19.541495182Z       - alertmanager:9093
2024-04-03T03:27:19.541498215Z global:
2024-04-03T03:27:19.541501206Z   evaluation_interval: 15s
2024-04-03T03:27:19.541504082Z   scrape_interval: 15s
2024-04-03T03:27:19.541507011Z rule_files:
2024-04-03T03:27:19.541526558Z - osm_metric_rules.yml
2024-04-03T03:27:19.541529817Z - osm_alert_rules.yml
2024-04-03T03:27:19.541532894Z scrape_configs:
2024-04-03T03:27:19.541535984Z - job_name: mon_exporter
2024-04-03T03:27:19.541538930Z   static_configs:
2024-04-03T03:27:19.541542380Z   - targets:
2024-04-03T03:27:19.541545417Z     - mon:8000
2024-04-03T03:27:19.541548513Z - honor_labels: true
2024-04-03T03:27:19.541551630Z   job_name: pushgateway
2024-04-03T03:27:19.541554618Z   scrape_interval: 30s
2024-04-03T03:27:19.541557633Z   static_configs:
2024-04-03T03:27:19.541560731Z   - targets:
2024-04-03T03:27:19.541564302Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:27:19.541567414Z 
2024-04-03T03:27:19.545623098Z Alerts read: 2
2024-04-03T03:27:19.548453138Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:27:19.551619630Z groups:
2024-04-03T03:27:19.551633170Z - name: osm_alert_rules
2024-04-03T03:27:19.551637164Z   rules:
2024-04-03T03:27:19.551640915Z   - alert: vdu_down
2024-04-03T03:27:19.551644455Z     annotations:
2024-04-03T03:27:19.551666656Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:27:19.551674883Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:27:19.551678616Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:27:19.551684338Z     expr: vm_status_extended != 1
2024-04-03T03:27:19.551688011Z     for: 3m
2024-04-03T03:27:19.551691913Z - name: _osm_alert_rules_20240403032719_
2024-04-03T03:27:19.551695505Z   rules:
2024-04-03T03:27:19.551700869Z   - alert: scalein_a438e133-20bb-4d40-bc7c-8359c4006bf2
2024-04-03T03:27:19.551704672Z     expr: (count (osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:19.551713475Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:19.551717636Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T03:27:19.551721424Z     for: 1m
2024-04-03T03:27:19.551724977Z     labels:
2024-04-03T03:27:19.551728563Z       ns_id: aa5dac12-c347-43c8-b4ce-bfa740b9f1b3
2024-04-03T03:27:19.551732147Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:27:19.551737539Z       vnf_member_index: vnf
2024-04-03T03:27:19.551741148Z   - alert: scaleout_1ff16aa4-c0be-4101-b6ea-126902dfc530
2024-04-03T03:27:19.551744673Z     expr: (count (osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:19.551748488Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="aa5dac12-c347-43c8-b4ce-bfa740b9f1b3",
2024-04-03T03:27:19.551765195Z       vnf_member_index="vnf", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T03:27:19.551774164Z     for: 1m
2024-04-03T03:27:19.551777699Z     labels:
2024-04-03T03:27:19.551781242Z       ns_id: aa5dac12-c347-43c8-b4ce-bfa740b9f1b3
2024-04-03T03:27:19.551784805Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T03:27:19.551788383Z       vnf_member_index: vnf
2024-04-03T03:27:19.551791736Z 
2024-04-03T03:27:19.555169334Z New config written, updating prometheus
2024-04-03T03:27:24.586472511Z Prometheus config update successful
2024-04-03T03:27:24.586509221Z 
2024-04-03T03:44:47.498452093Z Changes detected, updating prometheus config
2024-04-03T03:44:47.500249219Z Jobs detected: 0
2024-04-03T03:44:47.506199733Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:44:47.508463251Z alerting:
2024-04-03T03:44:47.508492787Z   alertmanagers:
2024-04-03T03:44:47.508503000Z   - static_configs:
2024-04-03T03:44:47.508511900Z     - targets:
2024-04-03T03:44:47.508521596Z       - alertmanager:9093
2024-04-03T03:44:47.508530886Z global:
2024-04-03T03:44:47.508539577Z   evaluation_interval: 15s
2024-04-03T03:44:47.508548060Z   scrape_interval: 15s
2024-04-03T03:44:47.508556793Z rule_files:
2024-04-03T03:44:47.508565071Z - osm_metric_rules.yml
2024-04-03T03:44:47.508573574Z - osm_alert_rules.yml
2024-04-03T03:44:47.508582112Z scrape_configs:
2024-04-03T03:44:47.508598664Z - job_name: mon_exporter
2024-04-03T03:44:47.508624645Z   static_configs:
2024-04-03T03:44:47.508633517Z   - targets:
2024-04-03T03:44:47.508642180Z     - mon:8000
2024-04-03T03:44:47.508650665Z - honor_labels: true
2024-04-03T03:44:47.508659372Z   job_name: pushgateway
2024-04-03T03:44:47.508668311Z   scrape_interval: 30s
2024-04-03T03:44:47.508676985Z   static_configs:
2024-04-03T03:44:47.508685433Z   - targets:
2024-04-03T03:44:47.508722777Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:44:47.508731581Z 
2024-04-03T03:44:47.511794806Z Alerts read: 0
2024-04-03T03:44:47.513757713Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:44:47.514705181Z groups:
2024-04-03T03:44:47.514739174Z - name: osm_alert_rules
2024-04-03T03:44:47.514747230Z   rules:
2024-04-03T03:44:47.514754627Z   - alert: vdu_down
2024-04-03T03:44:47.514761810Z     annotations:
2024-04-03T03:44:47.514770300Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:44:47.514778450Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:44:47.514785831Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:44:47.514794372Z     expr: vm_status_extended != 1
2024-04-03T03:44:47.514801788Z     for: 3m
2024-04-03T03:44:47.514808530Z 
2024-04-03T03:44:47.515661887Z New config written, updating prometheus
2024-04-03T03:44:52.542024982Z Prometheus config update successful
2024-04-03T03:44:52.542175763Z 
2024-04-03T03:44:52.542431024Z Changes detected, updating prometheus config
2024-04-03T03:44:52.544968811Z Jobs detected: 0
2024-04-03T03:44:52.552632678Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:44:52.554579530Z alerting:
2024-04-03T03:44:52.554683694Z   alertmanagers:
2024-04-03T03:44:52.554691253Z   - static_configs:
2024-04-03T03:44:52.554697102Z     - targets:
2024-04-03T03:44:52.554703672Z       - alertmanager:9093
2024-04-03T03:44:52.554732447Z global:
2024-04-03T03:44:52.554738527Z   evaluation_interval: 15s
2024-04-03T03:44:52.554744059Z   scrape_interval: 15s
2024-04-03T03:44:52.554749760Z rule_files:
2024-04-03T03:44:52.554755488Z - osm_metric_rules.yml
2024-04-03T03:44:52.554761503Z - osm_alert_rules.yml
2024-04-03T03:44:52.554767316Z scrape_configs:
2024-04-03T03:44:52.554772977Z - job_name: mon_exporter
2024-04-03T03:44:52.554778609Z   static_configs:
2024-04-03T03:44:52.554784147Z   - targets:
2024-04-03T03:44:52.554789978Z     - mon:8000
2024-04-03T03:44:52.554795566Z - honor_labels: true
2024-04-03T03:44:52.554801203Z   job_name: pushgateway
2024-04-03T03:44:52.554806889Z   scrape_interval: 30s
2024-04-03T03:44:52.554812367Z   static_configs:
2024-04-03T03:44:52.554817977Z   - targets:
2024-04-03T03:44:52.554824717Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:44:52.554830447Z 
2024-04-03T03:44:52.557459897Z Alerts read: 0
2024-04-03T03:44:52.559471700Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:44:52.560748706Z groups:
2024-04-03T03:44:52.560759388Z - name: osm_alert_rules
2024-04-03T03:44:52.560762712Z   rules:
2024-04-03T03:44:52.560765965Z   - alert: vdu_down
2024-04-03T03:44:52.560769324Z     annotations:
2024-04-03T03:44:52.560774010Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:44:52.560778033Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:44:52.560781372Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:44:52.560785140Z     expr: vm_status_extended != 1
2024-04-03T03:44:52.560788171Z     for: 3m
2024-04-03T03:44:52.560836248Z 
2024-04-03T03:44:52.562144871Z New config written, updating prometheus
2024-04-03T03:44:57.589904141Z Prometheus config update successful
2024-04-03T03:44:57.590000219Z 
2024-04-03T03:47:04.557266048Z Changes detected, updating prometheus config
2024-04-03T03:47:04.562413196Z Jobs detected: 0
2024-04-03T03:47:04.567918779Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:47:04.570108220Z alerting:
2024-04-03T03:47:04.570120147Z   alertmanagers:
2024-04-03T03:47:04.570124580Z   - static_configs:
2024-04-03T03:47:04.570127658Z     - targets:
2024-04-03T03:47:04.570133339Z       - alertmanager:9093
2024-04-03T03:47:04.570136325Z global:
2024-04-03T03:47:04.570139356Z   evaluation_interval: 15s
2024-04-03T03:47:04.570142343Z   scrape_interval: 15s
2024-04-03T03:47:04.570145300Z rule_files:
2024-04-03T03:47:04.570148338Z - osm_metric_rules.yml
2024-04-03T03:47:04.570151315Z - osm_alert_rules.yml
2024-04-03T03:47:04.570154237Z scrape_configs:
2024-04-03T03:47:04.570169394Z - job_name: mon_exporter
2024-04-03T03:47:04.570172421Z   static_configs:
2024-04-03T03:47:04.570175362Z   - targets:
2024-04-03T03:47:04.570178371Z     - mon:8000
2024-04-03T03:47:04.570181274Z - honor_labels: true
2024-04-03T03:47:04.570184241Z   job_name: pushgateway
2024-04-03T03:47:04.570187194Z   scrape_interval: 30s
2024-04-03T03:47:04.570190107Z   static_configs:
2024-04-03T03:47:04.570193088Z   - targets:
2024-04-03T03:47:04.570197924Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:47:04.570406141Z 
2024-04-03T03:47:04.574460904Z Alerts read: 1
2024-04-03T03:47:04.576627534Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:47:04.578823850Z groups:
2024-04-03T03:47:04.578834746Z - name: osm_alert_rules
2024-04-03T03:47:04.578856812Z   rules:
2024-04-03T03:47:04.578859833Z   - alert: vdu_down
2024-04-03T03:47:04.578862744Z     annotations:
2024-04-03T03:47:04.578866201Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:47:04.578872163Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:47:04.578875103Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:47:04.578891466Z     expr: vm_status_extended != 1
2024-04-03T03:47:04.578894500Z     for: 3m
2024-04-03T03:47:04.578897408Z - name: _osm_alert_rules_20240403034704_
2024-04-03T03:47:04.578900213Z   rules:
2024-04-03T03:47:04.578904315Z   - alert: vdu_alarm_1237bf6d-a6f4-4edb-b39e-d4e750d0a47b
2024-04-03T03:47:04.578909008Z     expr: osm_cpu_utilization{ns_id="4c1a4009-bf35-4a0b-b37d-75f8cd28a6ac", vnf_member_index="vnf",
2024-04-03T03:47:04.578912006Z       vdu_id="cirros_vnfd-VM"} < 20.0000
2024-04-03T03:47:04.578914749Z     for: 1m
2024-04-03T03:47:04.578917497Z     labels:
2024-04-03T03:47:04.578920261Z       ns_id: 4c1a4009-bf35-4a0b-b37d-75f8cd28a6ac
2024-04-03T03:47:04.578923363Z       vdu_id: cirros_vnfd-VM
2024-04-03T03:47:04.578926153Z       vdu_name: '{{ $labels.vdu_name }}'
2024-04-03T03:47:04.578928929Z       vnf_member_index: vnf
2024-04-03T03:47:04.579110006Z 
2024-04-03T03:47:04.581425014Z New config written, updating prometheus
2024-04-03T03:47:09.616718179Z Prometheus config update successful
2024-04-03T03:47:09.616856660Z 
2024-04-03T03:57:13.323314852Z Changes detected, updating prometheus config
2024-04-03T03:57:13.331752875Z Jobs detected: 0
2024-04-03T03:57:13.338098461Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:57:13.345318188Z alerting:
2024-04-03T03:57:13.345338764Z   alertmanagers:
2024-04-03T03:57:13.345342790Z   - static_configs:
2024-04-03T03:57:13.345346194Z     - targets:
2024-04-03T03:57:13.345350405Z       - alertmanager:9093
2024-04-03T03:57:13.345353551Z global:
2024-04-03T03:57:13.345356665Z   evaluation_interval: 15s
2024-04-03T03:57:13.345359851Z   scrape_interval: 15s
2024-04-03T03:57:13.345363267Z rule_files:
2024-04-03T03:57:13.345366426Z - osm_metric_rules.yml
2024-04-03T03:57:13.345369437Z - osm_alert_rules.yml
2024-04-03T03:57:13.345372617Z scrape_configs:
2024-04-03T03:57:13.345375824Z - job_name: mon_exporter
2024-04-03T03:57:13.345378890Z   static_configs:
2024-04-03T03:57:13.345381830Z   - targets:
2024-04-03T03:57:13.345385229Z     - mon:8000
2024-04-03T03:57:13.345388236Z - honor_labels: true
2024-04-03T03:57:13.345391232Z   job_name: pushgateway
2024-04-03T03:57:13.345394207Z   scrape_interval: 30s
2024-04-03T03:57:13.345397235Z   static_configs:
2024-04-03T03:57:13.345400261Z   - targets:
2024-04-03T03:57:13.345404147Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:57:13.345407148Z 
2024-04-03T03:57:13.348277153Z Alerts read: 0
2024-04-03T03:57:13.350465333Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:57:13.351699526Z groups:
2024-04-03T03:57:13.351711121Z - name: osm_alert_rules
2024-04-03T03:57:13.351714431Z   rules:
2024-04-03T03:57:13.351717405Z   - alert: vdu_down
2024-04-03T03:57:13.351720318Z     annotations:
2024-04-03T03:57:13.351723993Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:57:13.351727580Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:57:13.351753245Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:57:13.351770642Z     expr: vm_status_extended != 1
2024-04-03T03:57:13.351773750Z     for: 3m
2024-04-03T03:57:13.351776582Z 
2024-04-03T03:57:13.352862928Z New config written, updating prometheus
2024-04-03T03:57:18.389944444Z Prometheus config update successful
2024-04-03T03:57:18.390028015Z 
2024-04-03T03:59:34.234998637Z Changes detected, updating prometheus config
2024-04-03T03:59:34.237566461Z Jobs detected: 1
2024-04-03T03:59:34.243825905Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T03:59:34.247808810Z alerting:
2024-04-03T03:59:34.247836416Z   alertmanagers:
2024-04-03T03:59:34.247846420Z   - static_configs:
2024-04-03T03:59:34.247855126Z     - targets:
2024-04-03T03:59:34.247864571Z       - alertmanager:9093
2024-04-03T03:59:34.247873219Z global:
2024-04-03T03:59:34.247881919Z   evaluation_interval: 15s
2024-04-03T03:59:34.247890298Z   scrape_interval: 15s
2024-04-03T03:59:34.247899185Z rule_files:
2024-04-03T03:59:34.247907659Z - osm_metric_rules.yml
2024-04-03T03:59:34.247916061Z - osm_alert_rules.yml
2024-04-03T03:59:34.247924424Z scrape_configs:
2024-04-03T03:59:34.247932696Z - job_name: mon_exporter
2024-04-03T03:59:34.247941047Z   static_configs:
2024-04-03T03:59:34.247949492Z   - targets:
2024-04-03T03:59:34.247957887Z     - mon:8000
2024-04-03T03:59:34.247966959Z - honor_labels: true
2024-04-03T03:59:34.247975408Z   job_name: pushgateway
2024-04-03T03:59:34.247983706Z   scrape_interval: 30s
2024-04-03T03:59:34.247992045Z   static_configs:
2024-04-03T03:59:34.248000327Z   - targets:
2024-04-03T03:59:34.248010020Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T03:59:34.248018553Z - job_name: a646c7b9ede54003a96c66de34515ed5
2024-04-03T03:59:34.248026938Z   metrics_path: /snmp
2024-04-03T03:59:34.248035572Z   params:
2024-04-03T03:59:34.248044015Z     module:
2024-04-03T03:59:34.248070528Z     - osm-snmp
2024-04-03T03:59:34.248079099Z   relabel_configs:
2024-04-03T03:59:34.248087584Z   - source_labels:
2024-04-03T03:59:34.248095870Z     - __address__
2024-04-03T03:59:34.248104812Z     target_label: __param_target
2024-04-03T03:59:34.248113261Z   - source_labels:
2024-04-03T03:59:34.248121769Z     - __param_target
2024-04-03T03:59:34.248130086Z     target_label: instance
2024-04-03T03:59:34.248139336Z   - replacement: eechart-0061129135-snmpexporter.f364e7d6-2f0f-47d5-b7bf-19dfacee51cf.svc:80
2024-04-03T03:59:34.248147697Z     target_label: __address__
2024-04-03T03:59:34.248156011Z   static_configs:
2024-04-03T03:59:34.248164409Z   - targets:
2024-04-03T03:59:34.248179731Z     - 172.21.249.201
2024-04-03T03:59:34.248187904Z 
2024-04-03T03:59:34.252515504Z Alerts read: 0
2024-04-03T03:59:34.254436165Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T03:59:34.255599380Z groups:
2024-04-03T03:59:34.255618051Z - name: osm_alert_rules
2024-04-03T03:59:34.255626825Z   rules:
2024-04-03T03:59:34.255636286Z   - alert: vdu_down
2024-04-03T03:59:34.255644768Z     annotations:
2024-04-03T03:59:34.255654182Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T03:59:34.255663535Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T03:59:34.255671844Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T03:59:34.255681540Z     expr: vm_status_extended != 1
2024-04-03T03:59:34.255690142Z     for: 3m
2024-04-03T03:59:34.255698111Z 
2024-04-03T03:59:34.256637796Z New config written, updating prometheus
2024-04-03T03:59:39.296117401Z Prometheus config update successful
2024-04-03T03:59:39.301579473Z 
2024-04-03T04:00:47.124442628Z Changes detected, updating prometheus config
2024-04-03T04:00:47.126565531Z Jobs detected: 0
2024-04-03T04:00:47.132220953Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T04:00:47.134444725Z alerting:
2024-04-03T04:00:47.134455764Z   alertmanagers:
2024-04-03T04:00:47.134459683Z   - static_configs:
2024-04-03T04:00:47.134463107Z     - targets:
2024-04-03T04:00:47.134467124Z       - alertmanager:9093
2024-04-03T04:00:47.134470244Z global:
2024-04-03T04:00:47.134473294Z   evaluation_interval: 15s
2024-04-03T04:00:47.134476463Z   scrape_interval: 15s
2024-04-03T04:00:47.134479488Z rule_files:
2024-04-03T04:00:47.134482598Z - osm_metric_rules.yml
2024-04-03T04:00:47.134485692Z - osm_alert_rules.yml
2024-04-03T04:00:47.134488821Z scrape_configs:
2024-04-03T04:00:47.134491883Z - job_name: mon_exporter
2024-04-03T04:00:47.134495011Z   static_configs:
2024-04-03T04:00:47.134498106Z   - targets:
2024-04-03T04:00:47.134501160Z     - mon:8000
2024-04-03T04:00:47.134504230Z - honor_labels: true
2024-04-03T04:00:47.134507312Z   job_name: pushgateway
2024-04-03T04:00:47.134510412Z   scrape_interval: 30s
2024-04-03T04:00:47.134513689Z   static_configs:
2024-04-03T04:00:47.134516742Z   - targets:
2024-04-03T04:00:47.134520387Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T04:00:47.134523503Z 
2024-04-03T04:00:47.143035148Z Alerts read: 0
2024-04-03T04:00:47.145222536Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T04:00:47.146569627Z groups:
2024-04-03T04:00:47.146580226Z - name: osm_alert_rules
2024-04-03T04:00:47.146583624Z   rules:
2024-04-03T04:00:47.146586717Z   - alert: vdu_down
2024-04-03T04:00:47.146609608Z     annotations:
2024-04-03T04:00:47.146613237Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T04:00:47.146617491Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T04:00:47.146620588Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T04:00:47.146624095Z     expr: vm_status_extended != 1
2024-04-03T04:00:47.146627125Z     for: 3m
2024-04-03T04:00:47.146629900Z 
2024-04-03T04:00:47.147956882Z New config written, updating prometheus
2024-04-03T04:00:52.174657859Z Prometheus config update successful
2024-04-03T04:00:52.174688669Z 
2024-04-03T04:14:14.877453429Z Changes detected, updating prometheus config
2024-04-03T04:14:14.879437886Z Jobs detected: 0
2024-04-03T04:14:14.887923665Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T04:14:14.891220959Z alerting:
2024-04-03T04:14:14.891237268Z   alertmanagers:
2024-04-03T04:14:14.891242539Z   - static_configs:
2024-04-03T04:14:14.891246992Z     - targets:
2024-04-03T04:14:14.891298688Z       - alertmanager:9093
2024-04-03T04:14:14.891319638Z global:
2024-04-03T04:14:14.891323801Z   evaluation_interval: 15s
2024-04-03T04:14:14.891327947Z   scrape_interval: 15s
2024-04-03T04:14:14.891332137Z rule_files:
2024-04-03T04:14:14.891336245Z - osm_metric_rules.yml
2024-04-03T04:14:14.891340348Z - osm_alert_rules.yml
2024-04-03T04:14:14.891415055Z scrape_configs:
2024-04-03T04:14:14.891435740Z - job_name: mon_exporter
2024-04-03T04:14:14.891441004Z   static_configs:
2024-04-03T04:14:14.891444997Z   - targets:
2024-04-03T04:14:14.891448929Z     - mon:8000
2024-04-03T04:14:14.891452928Z - honor_labels: true
2024-04-03T04:14:14.891470783Z   job_name: pushgateway
2024-04-03T04:14:14.891474959Z   scrape_interval: 30s
2024-04-03T04:14:14.891479122Z   static_configs:
2024-04-03T04:14:14.891483094Z   - targets:
2024-04-03T04:14:14.891487893Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T04:14:14.891491820Z 
2024-04-03T04:14:14.898123149Z Alerts read: 2
2024-04-03T04:14:14.900684935Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T04:14:14.903841000Z groups:
2024-04-03T04:14:14.903856124Z - name: osm_alert_rules
2024-04-03T04:14:14.903860993Z   rules:
2024-04-03T04:14:14.903865339Z   - alert: vdu_down
2024-04-03T04:14:14.903869760Z     annotations:
2024-04-03T04:14:14.903875285Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T04:14:14.903879757Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T04:14:14.903883926Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T04:14:14.903927366Z     expr: vm_status_extended != 1
2024-04-03T04:14:14.903932354Z     for: 3m
2024-04-03T04:14:14.903940735Z - name: _osm_alert_rules_20240403041414_
2024-04-03T04:14:14.903945283Z   rules:
2024-04-03T04:14:14.903949896Z   - alert: scalein_e7b13033-5083-414b-a490-c56888564241
2024-04-03T04:14:14.903954689Z     expr: (count (osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:14.903959624Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:14.903978926Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T04:14:14.903983743Z     for: 1m
2024-04-03T04:14:14.903988030Z     labels:
2024-04-03T04:14:14.903992443Z       ns_id: 0f5dc377-1eee-4aee-ab5e-5a80b710e1e3
2024-04-03T04:14:14.903996712Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T04:14:14.904001621Z       vnf_member_index: '1'
2024-04-03T04:14:14.904005916Z   - alert: scaleout_19e4fe32-8ae7-4b64-801b-2b416e1b697e
2024-04-03T04:14:14.904010138Z     expr: (count (osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:14.904014561Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:14.904018814Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T04:14:14.904023118Z     for: 1m
2024-04-03T04:14:14.904027305Z     labels:
2024-04-03T04:14:14.904031518Z       ns_id: 0f5dc377-1eee-4aee-ab5e-5a80b710e1e3
2024-04-03T04:14:14.904035712Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T04:14:14.904039970Z       vnf_member_index: '1'
2024-04-03T04:14:14.904061404Z 
2024-04-03T04:14:14.907348721Z New config written, updating prometheus
2024-04-03T04:14:19.942065325Z Prometheus config update successful
2024-04-03T04:14:19.942133804Z 
2024-04-03T04:14:19.944236598Z Changes detected, updating prometheus config
2024-04-03T04:14:19.945551487Z Jobs detected: 0
2024-04-03T04:14:19.950738158Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T04:14:19.952901408Z alerting:
2024-04-03T04:14:19.952912591Z   alertmanagers:
2024-04-03T04:14:19.952918890Z   - static_configs:
2024-04-03T04:14:19.952921940Z     - targets:
2024-04-03T04:14:19.952927231Z       - alertmanager:9093
2024-04-03T04:14:19.952930353Z global:
2024-04-03T04:14:19.952933304Z   evaluation_interval: 15s
2024-04-03T04:14:19.952936551Z   scrape_interval: 15s
2024-04-03T04:14:19.952970163Z rule_files:
2024-04-03T04:14:19.952973431Z - osm_metric_rules.yml
2024-04-03T04:14:19.952976353Z - osm_alert_rules.yml
2024-04-03T04:14:19.952979583Z scrape_configs:
2024-04-03T04:14:19.952982665Z - job_name: mon_exporter
2024-04-03T04:14:19.952985539Z   static_configs:
2024-04-03T04:14:19.952988699Z   - targets:
2024-04-03T04:14:19.952991703Z     - mon:8000
2024-04-03T04:14:19.952994819Z - honor_labels: true
2024-04-03T04:14:19.952997873Z   job_name: pushgateway
2024-04-03T04:14:19.953000736Z   scrape_interval: 30s
2024-04-03T04:14:19.953003735Z   static_configs:
2024-04-03T04:14:19.953006769Z   - targets:
2024-04-03T04:14:19.953012116Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T04:14:19.955643972Z 
2024-04-03T04:14:19.961109277Z Alerts read: 2
2024-04-03T04:14:19.963909983Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T04:14:19.967446625Z groups:
2024-04-03T04:14:19.967473628Z - name: osm_alert_rules
2024-04-03T04:14:19.967477161Z   rules:
2024-04-03T04:14:19.967480070Z   - alert: vdu_down
2024-04-03T04:14:19.967483103Z     annotations:
2024-04-03T04:14:19.967488992Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T04:14:19.967494090Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T04:14:19.967497150Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T04:14:19.967501249Z     expr: vm_status_extended != 1
2024-04-03T04:14:19.967504229Z     for: 3m
2024-04-03T04:14:19.967507075Z - name: _osm_alert_rules_20240403041419_
2024-04-03T04:14:19.967509884Z   rules:
2024-04-03T04:14:19.967514735Z   - alert: scalein_e7b13033-5083-414b-a490-c56888564241
2024-04-03T04:14:19.967517934Z     expr: (count (osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:19.967523766Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) > 1) and (avg(osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:19.967526757Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) < 10.0000000000)
2024-04-03T04:14:19.967529902Z     for: 1m
2024-04-03T04:14:19.967533100Z     labels:
2024-04-03T04:14:19.967536102Z       ns_id: 0f5dc377-1eee-4aee-ab5e-5a80b710e1e3
2024-04-03T04:14:19.967538921Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T04:14:19.967544462Z       vnf_member_index: '1'
2024-04-03T04:14:19.967547440Z   - alert: scaleout_19e4fe32-8ae7-4b64-801b-2b416e1b697e
2024-04-03T04:14:19.967550506Z     expr: (count (osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:19.967553590Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) < 2) and (avg(osm_cpu_utilization{ns_id="0f5dc377-1eee-4aee-ab5e-5a80b710e1e3",
2024-04-03T04:14:19.967556511Z       vnf_member_index="1", vdu_id="hackfest_basic_metrics-VM"}) > 60.0000000000)
2024-04-03T04:14:19.967559414Z     for: 1m
2024-04-03T04:14:19.967562101Z     labels:
2024-04-03T04:14:19.967564860Z       ns_id: 0f5dc377-1eee-4aee-ab5e-5a80b710e1e3
2024-04-03T04:14:19.967567783Z       vdu_id: hackfest_basic_metrics-VM
2024-04-03T04:14:19.967570513Z       vnf_member_index: '1'
2024-04-03T04:14:19.967573105Z 
2024-04-03T04:14:19.970582995Z New config written, updating prometheus
2024-04-03T04:14:25.002033160Z Prometheus config update successful
2024-04-03T04:14:25.002068374Z 
2024-04-03T04:20:02.151141870Z Changes detected, updating prometheus config
2024-04-03T04:20:02.156073990Z Jobs detected: 0
2024-04-03T04:20:02.161572229Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T04:20:02.163864719Z alerting:
2024-04-03T04:20:02.163879843Z   alertmanagers:
2024-04-03T04:20:02.163883837Z   - static_configs:
2024-04-03T04:20:02.163887008Z     - targets:
2024-04-03T04:20:02.163891253Z       - alertmanager:9093
2024-04-03T04:20:02.163894678Z global:
2024-04-03T04:20:02.163897857Z   evaluation_interval: 15s
2024-04-03T04:20:02.163900996Z   scrape_interval: 15s
2024-04-03T04:20:02.163904116Z rule_files:
2024-04-03T04:20:02.163907115Z - osm_metric_rules.yml
2024-04-03T04:20:02.163910197Z - osm_alert_rules.yml
2024-04-03T04:20:02.163913249Z scrape_configs:
2024-04-03T04:20:02.163916257Z - job_name: mon_exporter
2024-04-03T04:20:02.163919330Z   static_configs:
2024-04-03T04:20:02.163922432Z   - targets:
2024-04-03T04:20:02.163925722Z     - mon:8000
2024-04-03T04:20:02.163928776Z - honor_labels: true
2024-04-03T04:20:02.163931754Z   job_name: pushgateway
2024-04-03T04:20:02.163934739Z   scrape_interval: 30s
2024-04-03T04:20:02.163938229Z   static_configs:
2024-04-03T04:20:02.163941354Z   - targets:
2024-04-03T04:20:02.163945304Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T04:20:02.163948190Z 
2024-04-03T04:20:02.172109849Z Alerts read: 0
2024-04-03T04:20:02.176116348Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T04:20:02.177328140Z groups:
2024-04-03T04:20:02.177340608Z - name: osm_alert_rules
2024-04-03T04:20:02.177344007Z   rules:
2024-04-03T04:20:02.177347528Z   - alert: vdu_down
2024-04-03T04:20:02.177350669Z     annotations:
2024-04-03T04:20:02.177354514Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T04:20:02.177361694Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T04:20:02.177364937Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T04:20:02.177368673Z     expr: vm_status_extended != 1
2024-04-03T04:20:02.177371799Z     for: 3m
2024-04-03T04:20:02.177374660Z 
2024-04-03T04:20:02.178555252Z New config written, updating prometheus
2024-04-03T04:20:07.205173670Z Prometheus config update successful
2024-04-03T04:20:07.205201852Z 
2024-04-03T04:20:07.205206281Z Changes detected, updating prometheus config
2024-04-03T04:20:07.206609097Z Jobs detected: 0
2024-04-03T04:20:07.211291757Z Writing new config file to /etc/prometheus/prometheus.yml
2024-04-03T04:20:07.213056359Z alerting:
2024-04-03T04:20:07.213066771Z   alertmanagers:
2024-04-03T04:20:07.213071588Z   - static_configs:
2024-04-03T04:20:07.213074979Z     - targets:
2024-04-03T04:20:07.213078778Z       - alertmanager:9093
2024-04-03T04:20:07.213081904Z global:
2024-04-03T04:20:07.213085183Z   evaluation_interval: 15s
2024-04-03T04:20:07.213088343Z   scrape_interval: 15s
2024-04-03T04:20:07.213091554Z rule_files:
2024-04-03T04:20:07.213094771Z - osm_metric_rules.yml
2024-04-03T04:20:07.213098147Z - osm_alert_rules.yml
2024-04-03T04:20:07.213101272Z scrape_configs:
2024-04-03T04:20:07.213104408Z - job_name: mon_exporter
2024-04-03T04:20:07.213107370Z   static_configs:
2024-04-03T04:20:07.213110528Z   - targets:
2024-04-03T04:20:07.213113675Z     - mon:8000
2024-04-03T04:20:07.213116793Z - honor_labels: true
2024-04-03T04:20:07.213119869Z   job_name: pushgateway
2024-04-03T04:20:07.213123243Z   scrape_interval: 30s
2024-04-03T04:20:07.213126719Z   static_configs:
2024-04-03T04:20:07.213141470Z   - targets:
2024-04-03T04:20:07.213145777Z     - pushgateway-prometheus-pushgateway:9091
2024-04-03T04:20:07.213164214Z 
2024-04-03T04:20:07.216192314Z Alerts read: 0
2024-04-03T04:20:07.218110954Z Writing new alerts file to /etc/prometheus/osm_alert_rules.yml
2024-04-03T04:20:07.219345368Z groups:
2024-04-03T04:20:07.219365014Z - name: osm_alert_rules
2024-04-03T04:20:07.219380034Z   rules:
2024-04-03T04:20:07.219384593Z   - alert: vdu_down
2024-04-03T04:20:07.219389108Z     annotations:
2024-04-03T04:20:07.219394167Z       description: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} has been down
2024-04-03T04:20:07.219399904Z         for more than 3 minutes. NS instance id is {{ $labels.ns_id }}
2024-04-03T04:20:07.219403238Z       summary: VDU {{ $labels.vm_id }} in VIM {{ $labels.vim_id }} is down
2024-04-03T04:20:07.219406902Z     expr: vm_status_extended != 1
2024-04-03T04:20:07.219409984Z     for: 3m
2024-04-03T04:20:07.219412810Z 
2024-04-03T04:20:07.220433577Z New config written, updating prometheus
2024-04-03T04:20:12.248165851Z Prometheus config update successful
2024-04-03T04:20:12.248188915Z