From 456d0f323cfcb5fe6b8cad5a6c3e6633875633cd Mon Sep 17 00:00:00 2001 From: lavado Date: Fri, 15 Nov 2019 17:04:02 -0500 Subject: [PATCH] Grafana dashboard automation Change-Id: I78a353ad59ea39cf747412e00ae2210826862150 Signed-off-by: lavado --- docker/scripts/start.sh | 25 +- osm_mon/cmd/mon_dashboarder.py | 54 ++ osm_mon/core/common_db.py | 19 +- osm_mon/core/mon.yaml | 8 + osm_mon/dashboarder/__init__ .py | 23 + osm_mon/dashboarder/backends/grafana.py | 67 ++ osm_mon/dashboarder/dashboarder.py | 53 ++ osm_mon/dashboarder/service.py | 93 +++ osm_mon/dashboarder/templates/ns_scoped.json | 646 ++++++++++++++++++ .../dashboarder/templates/project_scoped.json | 608 +++++++++++++++++ setup.py | 1 + 11 files changed, 1594 insertions(+), 3 deletions(-) create mode 100644 osm_mon/cmd/mon_dashboarder.py create mode 100644 osm_mon/dashboarder/__init__ .py create mode 100644 osm_mon/dashboarder/backends/grafana.py create mode 100644 osm_mon/dashboarder/dashboarder.py create mode 100644 osm_mon/dashboarder/service.py create mode 100644 osm_mon/dashboarder/templates/ns_scoped.json create mode 100644 osm_mon/dashboarder/templates/project_scoped.json diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh index 8e2880b..0b0cb79 100644 --- a/docker/scripts/start.sh +++ b/docker/scripts/start.sh @@ -1,3 +1,25 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: glavado@whitestack.com +## DB_EXISTS="" max_attempts=120 @@ -54,4 +76,5 @@ fi osm-mon-server & osm-mon-evaluator & -osm-mon-collector \ No newline at end of file +osm-mon-collector & +osm-mon-dashboarder \ No newline at end of file diff --git a/osm_mon/cmd/mon_dashboarder.py b/osm_mon/cmd/mon_dashboarder.py new file mode 100644 index 0000000..211038d --- /dev/null +++ b/osm_mon/cmd/mon_dashboarder.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import argparse +import logging +import sys + +from osm_mon.core.config import Config +from osm_mon.dashboarder.dashboarder import Dashboarder + + +def main(): + parser = argparse.ArgumentParser(prog='osm-policy-agent') + parser.add_argument('--config-file', nargs='?', help='POL configuration file') + args = parser.parse_args() + cfg = Config(args.config_file) + + root = logging.getLogger() + root.setLevel(logging.getLevelName(cfg.get('global', 'loglevel'))) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.getLevelName(cfg.get('global', 'loglevel'))) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', '%m/%d/%Y %I:%M:%S %p') + ch.setFormatter(formatter) + root.addHandler(ch) + + log = logging.getLogger(__name__) + log.info("Starting MON Dashboarder...") + log.debug("Config: %s", cfg.conf) + dashboarder = Dashboarder(cfg) + dashboarder.dashboard_forever() + + +if __name__ == '__main__': + main() diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py index 8df8672..7bd2491 100644 --- a/osm_mon/core/common_db.py +++ b/osm_mon/core/common_db.py @@ -55,9 +55,18 @@ class CommonDbClient: return vnfrs def get_vnfd(self, vnfd_id: str): - vnfr = self.common_db.get_one("vnfds", + vnfd = self.common_db.get_one("vnfds", {"_id": vnfd_id}) - return vnfr + return vnfd + + def get_vnfd_by_name(self, vnfd_name: str): + # TODO: optimize way of getting single VNFD in shared enviroments (RBAC) + vnfd = self.common_db.get_list("vnfds", + {"name": vnfd_name})[0] + return vnfd + + def get_nsrs(self): + return self.common_db.get_list('nsrs') def get_nsr(self, nsr_id: str): nsr = self.common_db.get_one("nsrs", @@ -115,3 +124,9 @@ class CommonDbClient: def get_sdnc(self, sdnc_id: str): return self.common_db.get_one('sdns', {'_id': sdnc_id}) + + def get_projects(self): + return self.common_db.get_list('projects') + + def get_project(self, project_id: str): + return self.common_db.get_one('projects', {'_id': project_id}) diff --git a/osm_mon/core/mon.yaml b/osm_mon/core/mon.yaml index c094dda..bfdda5a 100644 --- a/osm_mon/core/mon.yaml +++ b/osm_mon/core/mon.yaml @@ -46,6 +46,14 @@ evaluator: interval: 30 backend: prometheus +dashboarder: + interval: 30 + backend: grafana + +grafana: + url: http://grafana:3000 + token: "Basic YWRtaW46YWRtaW4=" + prometheus: url: http://prometheus:9090 diff --git a/osm_mon/dashboarder/__init__ .py b/osm_mon/dashboarder/__init__ .py new file mode 100644 index 0000000..542ae4c --- /dev/null +++ b/osm_mon/dashboarder/__init__ .py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: glavado@whitestack.com +## diff --git a/osm_mon/dashboarder/backends/grafana.py b/osm_mon/dashboarder/backends/grafana.py new file mode 100644 index 0000000..123f9e9 --- /dev/null +++ b/osm_mon/dashboarder/backends/grafana.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: glavado@whitestack.com +## +import logging +import requests + +log = logging.getLogger(__name__) + +# TODO (lavado): migrate to Class, import config variables +url = "http://grafana:3000/api/" +headers = { + 'content-type': "application/json", + 'authorization': "Basic YWRtaW46YWRtaW4=" + } + + +def get_all_dashboard_uids(): + response = requests.request("GET", url + "search?query=%", headers=headers) + dashboards = response.json() + dashboard_uids = [] + for dashboard in dashboards: + print(dashboard['uid']) + dashboard_uids.append(dashboard['uid']) + log.debug("Searching for all dashboard uids: %s", dashboard_uids) + return dashboard_uids + + +def get_dashboard_status(uid): + response = requests.request("GET", url + "dashboards/uid/" + uid, headers=headers) + log.debug("Searching for dashboard result: %s", response.text) + return response + + +def create_dashboard(uid, name, json_file): + with open(json_file) as f: + dashboard_data = f.read() + + dashboard_data = dashboard_data.replace('OSM_ID', uid).replace('OSM_NAME', name) + + response = requests.request("POST", url + "dashboards/db/", data=dashboard_data, headers=headers) + log.debug("Creating dashboard result: %s", response.text) + return response + + +def delete_dashboard(uid): + response = requests.request("DELETE", url + "dashboards/uid/" + uid, headers=headers) + log.debug("Delete dashboard result: %s", response.text) + return response diff --git a/osm_mon/dashboarder/dashboarder.py b/osm_mon/dashboarder/dashboarder.py new file mode 100644 index 0000000..dc21e33 --- /dev/null +++ b/osm_mon/dashboarder/dashboarder.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging +import time + +import peewee + +from osm_mon.dashboarder.service import DashboarderService +from osm_mon.core.config import Config + +log = logging.getLogger(__name__) + + +class Dashboarder: + def __init__(self, config: Config): + self.conf = config + self.service = DashboarderService(config) + + def dashboard_forever(self): + log.debug('dashboard_forever') + while True: + try: + self.create_dashboards() + time.sleep(int(self.conf.get('dashboarder', 'interval'))) + except peewee.PeeweeException: + log.exception("Database error consuming message: ") + raise + except Exception: + log.exception("Error creating dashboards") + + def create_dashboards(self): + self.service.create_dashboards() + log.debug('I just called the dashboarder service!') diff --git a/osm_mon/dashboarder/service.py b/osm_mon/dashboarder/service.py new file mode 100644 index 0000000..10dc0f7 --- /dev/null +++ b/osm_mon/dashboarder/service.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: glavado@whitestack.com +## +import logging + +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +import osm_mon.dashboarder.backends.grafana as grafana + +log = logging.getLogger(__name__) + + +class DashboarderService: + def __init__(self, config: Config): + self.conf = config + self.common_db = CommonDbClient(self.conf) + + def create_dashboards(self): + # TODO lavado: migrate these methods to mongo change streams + # Lists all dashboards and OSM resources for later comparisons + dashboard_uids = grafana.get_all_dashboard_uids() + osm_resource_uids = [] + + # Reads existing project list and creates a dashboard for each + projects = self.common_db.get_projects() + for project in projects: + project_id = project['_id'] + # Collect Project IDs for periodical dashboard clean-up + osm_resource_uids.append(project_id) + dashboard_path = '/mon/osm_mon/dashboarder/templates/project_scoped.json' + if project_id not in dashboard_uids: + project_name = project['name'] + grafana.create_dashboard(project_id, project_name, + dashboard_path) + log.debug('Created dashboard for Project: %s', project_id) + else: + log.debug('Dashboard already exists') + + # Reads existing NS list and creates a dashboard for each + # TODO lavado: only create for ACTIVE NSRs + nsrs = self.common_db.get_nsrs() + for nsr in nsrs: + nsr_id = nsr['_id'] + dashboard_path = '/mon/osm_mon/dashboarder/templates/ns_scoped.json' + # Collect NS IDs for periodical dashboard clean-up + osm_resource_uids.append(nsr_id) + # Check if the NSR's VNFDs contain metrics + constituent_vnfds = nsr['nsd']['constituent-vnfd'] + for constituent_vnfd in constituent_vnfds: + try: + vnfd = self.common_db.get_vnfd_by_name(constituent_vnfd['vnfd-id-ref']) + # If there are metrics, create dashboard (if exists) + if 'monitoring-param' in vnfd: + if nsr_id not in dashboard_uids: + nsr_name = nsr['name'] + grafana.create_dashboard(nsr_id, nsr_name, + dashboard_path) + log.debug('Created dashboard for NS: %s', nsr_id) + else: + log.debug('Dashboard already exists') + break + else: + log.debug('NS does not has metrics') + except Exception: + log.exception("VNFD is not valid or has been renamed") + continue + + # Delete obsolete dashboards + for dashboard_uid in dashboard_uids: + if dashboard_uid not in osm_resource_uids: + grafana.delete_dashboard(dashboard_uid) + log.debug('Deleted obsolete dashboard: %s', dashboard_uid) + else: + log.debug('All dashboards in use') diff --git a/osm_mon/dashboarder/templates/ns_scoped.json b/osm_mon/dashboarder/templates/ns_scoped.json new file mode 100644 index 0000000..693d2e1 --- /dev/null +++ b/osm_mon/dashboarder/templates/ns_scoped.json @@ -0,0 +1,646 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "VIM Metrics", + "type": "row" + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "decimals": 2, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_cpu_utilization{ns_id=\"$ns_id\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "NS {{ns_name}} - VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "decimals": 2, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_average_memory_utilization{ns_id=\"$ns_id\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "NS {{ns_name}} - VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "decmbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "fill": 1, + "fillGradient": 3, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 8 + }, + "height": "300", + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*_sent$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_packets_received{ns_id=\"$ns_id\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "A", + "step": 4 + }, + { + "expr": "osm_packets_sent{ns_id=\"$ns_id\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "transmit (-) / receive (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 14, + "panels": [], + "title": "VNF Indicators", + "type": "row" + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "decimals": 2, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_load{ns_id=\"$ns_id\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "decimals": 2, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_users{ns_id=\"$ns_id\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "VNF {{vnf_member_index}} - VDU {{vdu_name}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Users", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "OSM_ID", + "value": "OSM_ID" + }, + "hide": 2, + "label": "NS ID", + "name": "ns_id", + "options": [ + { + "selected": true, + "text": "OSM_ID", + "value": "OSM_ID" + } + ], + "query": "OSM_ID", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OSM NS Metrics - OSM_NAME", + "uid": "OSM_ID", + "version": 1 +} +} \ No newline at end of file diff --git a/osm_mon/dashboarder/templates/project_scoped.json b/osm_mon/dashboarder/templates/project_scoped.json new file mode 100644 index 0000000..6c788d0 --- /dev/null +++ b/osm_mon/dashboarder/templates/project_scoped.json @@ -0,0 +1,608 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1573641787428, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "osm_prometheus", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 8, + "links": [], + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Status", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "Value", + "sanitize": false, + "thresholds": [ + "0.1", + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "UP", + "value": "1" + }, + { + "text": "DOWN", + "value": "0" + } + ] + }, + { + "alias": "VIM ID", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "osm_vim_status{project_id=\"$project_id\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{vim_account_id}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "VIM Status", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "columns": [], + "datasource": "osm_prometheus", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 9, + "links": [], + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Status", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "link": false, + "mappingType": 1, + "pattern": "Value", + "sanitize": false, + "thresholds": [ + "0.1", + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "UP", + "value": "1" + }, + { + "text": "DOWN", + "value": "0" + } + ] + }, + { + "alias": "SDN Controller ID", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "osm_sdnc_status{project_id=\"$project_id\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{sdnc_id}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "SDN Controller Status", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "osm_prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(max(osm_vm_status{project_id=\"$project_id\"}) by (ns_name))", + "instant": true, + "refId": "A" + } + ], + "thresholds": "0.1,1000", + "timeFrom": null, + "timeShift": null, + "title": "Network Services", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "datasource": "osm_prometheus", + "gridPos": { + "h": 6, + "w": 8, + "x": 4, + "y": 5 + }, + "id": 11, + "links": [], + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": [ + "sum" + ], + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": [ + { + "color": "yellow", + "value": null + }, + { + "color": "red", + "value": 80 + } + ], + "title": "" + }, + "override": {}, + "values": true + }, + "orientation": "horizontal" + }, + "pluginVersion": "6.4.4", + "targets": [ + { + "expr": "count(osm_vm_status{project_id=\"$project_id\"}) by (ns_name)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "NS {{nsr_id}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "VDUs per NS", + "type": "bargauge" + }, + { + "columns": [], + "datasource": "osm_prometheus", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "links": [], + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "[Time]*[__name__]*[instance]*[job]*", + "type": "hidden" + }, + { + "alias": "Status", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "Value", + "sanitize": false, + "thresholds": [ + "0.1", + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "UP", + "value": "1" + }, + { + "text": "DOWN", + "value": "0" + } + ] + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM/DD/YY h:mm:ss a", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "date", + "unit": "short" + } + ], + "targets": [ + { + "expr": "min(osm_vm_status{project_id=\"$project_id\"}) by (ns_name)", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "NS {{nsr_id}} - VNF {{vnf_member_index}} - VDU {{vdur_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Service Status (based on VM State)", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "osm_vm_status{project_id=\"$project_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nsr_id}} - {{vdur_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VDU Status over time", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": "Status", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "OSM_ID", + "value": "OSM_ID" + }, + "hide": 2, + "label": "Project ID", + "name": "project_id", + "options": [ + { + "selected": true, + "text": "OSM_ID", + "value": "OSM_ID" + } + ], + "query": "OSM_ID", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OSM Project Status - OSM_NAME", + "uid": "OSM_ID", + "version": 2 +} +} \ No newline at end of file diff --git a/setup.py b/setup.py index 1cee33f..37a2566 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ setup( "osm-mon-server = osm_mon.cmd.mon_server:main", "osm-mon-evaluator = osm_mon.cmd.mon_evaluator:main", "osm-mon-collector = osm_mon.cmd.mon_collector:main", + "osm-mon-dashboarder = osm_mon.cmd.mon_dashboarder:main", "osm-mon-healthcheck = osm_mon.cmd.mon_healthcheck:main", ] }, -- 2.25.1