From 6332d381ac94fcee447d24fa33c15997c21bbf79 Mon Sep 17 00:00:00 2001 From: sousaedu Date: Thu, 25 Feb 2021 23:24:47 +0100 Subject: [PATCH] Adding scraping relation to Kafka Exporter Change-Id: Iea3c8f39f8a90fb4e287b074bf06f1042b71857c Signed-off-by: sousaedu --- .../.gitignore | 2 +- .../.yamllint.yaml | 0 .../README.md | 0 .../config.yaml | 0 .../files/kafka_exporter_dashboard.json | 609 ++++++++++++++++++ .../metadata.yaml | 6 +- .../requirements.txt | 0 .../src/charm.py | 31 +- .../src/pod_spec.py | 8 +- .../tests/__init__.py | 0 .../tests/test_charm.py | 93 +-- .../tests/test_pod_spec.py | 32 +- .../tox.ini | 6 +- .../files/kafka_exporter_dashboard.yaml | 543 ---------------- 14 files changed, 710 insertions(+), 620 deletions(-) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/.gitignore (96%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/.yamllint.yaml (100%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/README.md (100%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/config.yaml (100%) create mode 100644 installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/metadata.yaml (94%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/requirements.txt (100%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/src/charm.py (88%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/src/pod_spec.py (97%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/tests/__init__.py (100%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/tests/test_charm.py (85%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/tests/test_pod_spec.py (95%) rename installers/charm/{prometheus-kafka-exporter => kafka-exporter}/tox.ini (93%) delete mode 100644 installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml diff --git a/installers/charm/prometheus-kafka-exporter/.gitignore b/installers/charm/kafka-exporter/.gitignore similarity index 96% rename from installers/charm/prometheus-kafka-exporter/.gitignore rename to installers/charm/kafka-exporter/.gitignore index d1c82184..0933edca 100644 --- a/installers/charm/prometheus-kafka-exporter/.gitignore +++ b/installers/charm/kafka-exporter/.gitignore @@ -22,7 +22,7 @@ venv .vscode build -prometheus-kafka-exporter.charm +kafka-exporter.charm .coverage .stestr cover diff --git a/installers/charm/prometheus-kafka-exporter/.yamllint.yaml b/installers/charm/kafka-exporter/.yamllint.yaml similarity index 100% rename from installers/charm/prometheus-kafka-exporter/.yamllint.yaml rename to installers/charm/kafka-exporter/.yamllint.yaml diff --git a/installers/charm/prometheus-kafka-exporter/README.md b/installers/charm/kafka-exporter/README.md similarity index 100% rename from installers/charm/prometheus-kafka-exporter/README.md rename to installers/charm/kafka-exporter/README.md diff --git a/installers/charm/prometheus-kafka-exporter/config.yaml b/installers/charm/kafka-exporter/config.yaml similarity index 100% rename from installers/charm/prometheus-kafka-exporter/config.yaml rename to installers/charm/kafka-exporter/config.yaml diff --git a/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json b/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json new file mode 100644 index 00000000..5b7552ad --- /dev/null +++ b/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json @@ -0,0 +1,609 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Kafka resource usage and throughput", + "editable": true, + "gnetId": 7589, + "graphTooltip": 0, + "id": 10, + "iteration": 1578848023483, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus - Juju generated source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_topic_partition_current_offset - kafka_topic_partition_oldest_offset{instance=\"$instance\", topic=~\"$topic\"}) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages stored per topic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus - Juju generated source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 10, + "y": 0 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kafka_consumergroup_lag{instance=\"$instance\",topic=~\"$topic\"}) by (consumergroup, topic) ", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{topic}} ({{consumergroup}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lag by Consumer Group", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus - Juju generated source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 10 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(kafka_topic_partition_current_offset{instance=~'$instance', topic=~\"$topic\"}[5m])/5) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages produced per minute", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus - Juju generated source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 10, + "y": 10 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 480, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[5m])/5) by (consumergroup, topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " {{topic}} ({{consumergroup}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages consumed per minute", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus - Juju generated source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 420, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(topic) (kafka_topic_partitions{instance=\"$instance\",topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Partitions per Topic", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 19, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "osm-kafka-exporter-service", + "value": "osm-kafka-exporter-service" + }, + "datasource": "prometheus - Juju generated source", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(kafka_consumergroup_current_offset, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "datasource": "prometheus - Juju generated source", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(kafka_consumergroup_current_offset{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "prometheus - Juju generated source", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Topic", + "multi": true, + "name": "topic", + "options": [], + "query": "label_values(kafka_topic_partition_current_offset{instance='$instance',topic!='__consumer_offsets',topic!='--kafka'}, topic)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "topic", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka", + "uid": "jwPKIsniz", + "version": 2 +} diff --git a/installers/charm/prometheus-kafka-exporter/metadata.yaml b/installers/charm/kafka-exporter/metadata.yaml similarity index 94% rename from installers/charm/prometheus-kafka-exporter/metadata.yaml rename to installers/charm/kafka-exporter/metadata.yaml index 55f2ab0f..1339d279 100644 --- a/installers/charm/prometheus-kafka-exporter/metadata.yaml +++ b/installers/charm/kafka-exporter/metadata.yaml @@ -19,7 +19,7 @@ # osm-charmers@lists.launchpad.net ## -name: prometheus-kafka-exporter +name: kafka-exporter summary: OSM Prometheus Kafka Exporter description: | A CAAS charm to deploy OSM's Prometheus Kafka Exporter. @@ -43,7 +43,7 @@ requires: kafka: interface: kafka provides: - prometheus-target: - interface: http + prometheus-scrape: + interface: prometheus grafana-dashboard: interface: grafana-dashboard diff --git a/installers/charm/prometheus-kafka-exporter/requirements.txt b/installers/charm/kafka-exporter/requirements.txt similarity index 100% rename from installers/charm/prometheus-kafka-exporter/requirements.txt rename to installers/charm/kafka-exporter/requirements.txt diff --git a/installers/charm/prometheus-kafka-exporter/src/charm.py b/installers/charm/kafka-exporter/src/charm.py similarity index 88% rename from installers/charm/prometheus-kafka-exporter/src/charm.py rename to installers/charm/kafka-exporter/src/charm.py index 70a31f85..9f03a343 100755 --- a/installers/charm/prometheus-kafka-exporter/src/charm.py +++ b/installers/charm/kafka-exporter/src/charm.py @@ -23,6 +23,7 @@ import logging from pathlib import Path from typing import Dict, List, NoReturn +from urllib.parse import urlparse from ops.charm import CharmBase from ops.framework import EventBase, StoredState @@ -34,7 +35,7 @@ from pod_spec import make_pod_spec logger = logging.getLogger(__name__) -PROMETHEUS_KAFKA_EXPORTER_PORT = 9308 +KAFKA_EXPORTER_PORT = 9308 class RelationsMissing(Exception): @@ -93,19 +94,19 @@ def get_relation_data( return data -class PrometheusKafkaExporterCharm(CharmBase): - """Prometheus Kafka Exporter Charm.""" +class KafkaExporterCharm(CharmBase): + """Kafka Exporter Charm.""" state = StoredState() def __init__(self, *args) -> NoReturn: - """Prometheus Kafka Exporter Charm constructor.""" + """Kafka Exporter Charm constructor.""" super().__init__(*args) # Internal state initialization self.state.set_default(pod_spec=None) - self.port = PROMETHEUS_KAFKA_EXPORTER_PORT + self.port = KAFKA_EXPORTER_PORT self.image = OCIImageResource(self, "image") # Registering regular events @@ -120,21 +121,26 @@ class PrometheusKafkaExporterCharm(CharmBase): # Registering provided relation events self.framework.observe( - self.on.prometheus_target_relation_joined, self._publish_target_info + self.on.prometheus_scrape_relation_joined, self._publish_scrape_info ) self.framework.observe( self.on.grafana_dashboard_relation_joined, self._publish_dashboard_info ) - def _publish_target_info(self, event: EventBase) -> NoReturn: - """Publishes target information. + def _publish_scrape_info(self, event: EventBase) -> NoReturn: + """Publishes scrape information. Args: event (EventBase): Exporter relation event. """ rel_data = { - "hostname": self.model.app.name, - "port": str(PROMETHEUS_KAFKA_EXPORTER_PORT), + "hostname": urlparse(self.model.config["site_url"]).hostname + if self.model.config["site_url"] + else self.model.app.name, + "port": "80" if self.model.config["site_url"] else str(KAFKA_EXPORTER_PORT), + "metrics_path": "/metrics", + "scrape_interval": "30s", + "scrape_timeout": "15s", } for k, v in rel_data.items(): event.relation.data[self.unit][k] = v @@ -146,7 +152,8 @@ class PrometheusKafkaExporterCharm(CharmBase): event (EventBase): Exporter relation event. """ rel_data = { - "dashboard": Path("files/kafka_exporter_dashboard.yaml").read_text(), + "name": "osm-kafka", + "dashboard": Path("files/kafka_exporter_dashboard.json").read_text(), } for k, v in rel_data.items(): event.relation.data[self.unit][k] = v @@ -213,4 +220,4 @@ class PrometheusKafkaExporterCharm(CharmBase): if __name__ == "__main__": - main(PrometheusKafkaExporterCharm) + main(KafkaExporterCharm) diff --git a/installers/charm/prometheus-kafka-exporter/src/pod_spec.py b/installers/charm/kafka-exporter/src/pod_spec.py similarity index 97% rename from installers/charm/prometheus-kafka-exporter/src/pod_spec.py rename to installers/charm/kafka-exporter/src/pod_spec.py index 256c2033..a50d96f6 100644 --- a/installers/charm/prometheus-kafka-exporter/src/pod_spec.py +++ b/installers/charm/kafka-exporter/src/pod_spec.py @@ -102,9 +102,7 @@ def _make_pod_ports(port: int) -> List[Dict[str, Any]]: Returns: List[Dict[str, Any]]: pod port details. """ - return [ - {"name": "prometheus-kafka-exporter", "containerPort": port, "protocol": "TCP"} - ] + return [{"name": "kafka-exporter", "containerPort": port, "protocol": "TCP"}] def _make_pod_envconfig( @@ -246,7 +244,7 @@ def _make_pod_command(relation: Dict[str, Any]) -> List[str]: List[str]: command to startup the process. """ command = [ - "kafka-exporter", + "kafka_exporter", "--kafka.server={}:{}".format( relation.get("kafka_host"), relation.get("kafka_port") ), @@ -259,7 +257,7 @@ def make_pod_spec( image_info: Dict[str, str], config: Dict[str, Any], relation_state: Dict[str, Any], - app_name: str = "prometheus-kafka-exporter", + app_name: str = "kafka-exporter", port: int = 9308, ) -> Dict[str, Any]: """Generate the pod spec information. diff --git a/installers/charm/prometheus-kafka-exporter/tests/__init__.py b/installers/charm/kafka-exporter/tests/__init__.py similarity index 100% rename from installers/charm/prometheus-kafka-exporter/tests/__init__.py rename to installers/charm/kafka-exporter/tests/__init__.py diff --git a/installers/charm/prometheus-kafka-exporter/tests/test_charm.py b/installers/charm/kafka-exporter/tests/test_charm.py similarity index 85% rename from installers/charm/prometheus-kafka-exporter/tests/test_charm.py rename to installers/charm/kafka-exporter/tests/test_charm.py index 6b508748..53613214 100644 --- a/installers/charm/prometheus-kafka-exporter/tests/test_charm.py +++ b/installers/charm/kafka-exporter/tests/test_charm.py @@ -26,15 +26,15 @@ import unittest from ops.model import BlockedStatus from ops.testing import Harness -from charm import PrometheusKafkaExporterCharm +from charm import KafkaExporterCharm class TestCharm(unittest.TestCase): - """Prometheus Kafka Exporter Charm unit tests.""" + """Kafka Exporter Charm unit tests.""" def setUp(self) -> NoReturn: """Test setup""" - self.harness = Harness(PrometheusKafkaExporterCharm) + self.harness = Harness(KafkaExporterCharm) self.harness.set_leader(is_leader=True) self.harness.begin() @@ -59,12 +59,12 @@ class TestCharm(unittest.TestCase): "version": 3, "containers": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "imageDetails": self.harness.charm.image.fetch(), "imagePullPolicy": "Always", "ports": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "containerPort": 9308, "protocol": "TCP", } @@ -125,12 +125,12 @@ class TestCharm(unittest.TestCase): "version": 3, "containers": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "imageDetails": self.harness.charm.image.fetch(), "imagePullPolicy": "Always", "ports": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "containerPort": 9308, "protocol": "TCP", } @@ -164,20 +164,20 @@ class TestCharm(unittest.TestCase): "kubernetesResources": { "ingressResources": [ { - "name": "prometheus-kafka-exporter-ingress", + "name": "kafka-exporter-ingress", "annotations": { "nginx.ingress.kubernetes.io/ssl-redirect": "false", }, "spec": { "rules": [ { - "host": "prometheus-kafka-exporter", + "host": "kafka-exporter", "http": { "paths": [ { "path": "/", "backend": { - "serviceName": "prometheus-kafka-exporter", + "serviceName": "kafka-exporter", "servicePort": 9308, }, } @@ -205,7 +205,7 @@ class TestCharm(unittest.TestCase): }, ) - self.harness.update_config({"site_url": "http://prometheus-kafka-exporter"}) + self.harness.update_config({"site_url": "http://kafka-exporter"}) pod_spec, _ = self.harness.get_pod_spec() @@ -217,12 +217,12 @@ class TestCharm(unittest.TestCase): "version": 3, "containers": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "imageDetails": self.harness.charm.image.fetch(), "imagePullPolicy": "Always", "ports": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "containerPort": 9308, "protocol": "TCP", } @@ -256,18 +256,18 @@ class TestCharm(unittest.TestCase): "kubernetesResources": { "ingressResources": [ { - "name": "prometheus-kafka-exporter-ingress", + "name": "kafka-exporter-ingress", "annotations": {}, "spec": { "rules": [ { - "host": "prometheus-kafka-exporter", + "host": "kafka-exporter", "http": { "paths": [ { "path": "/", "backend": { - "serviceName": "prometheus-kafka-exporter", + "serviceName": "kafka-exporter", "servicePort": 9308, }, } @@ -277,8 +277,8 @@ class TestCharm(unittest.TestCase): ], "tls": [ { - "hosts": ["prometheus-kafka-exporter"], - "secretName": "prometheus-kafka-exporter", + "hosts": ["kafka-exporter"], + "secretName": "kafka-exporter", } ], }, @@ -303,8 +303,8 @@ class TestCharm(unittest.TestCase): self.harness.update_config( { - "site_url": "https://prometheus-kafka-exporter", - "tls_secret_name": "prometheus-kafka-exporter", + "site_url": "https://kafka-exporter", + "tls_secret_name": "kafka-exporter", } ) @@ -318,12 +318,12 @@ class TestCharm(unittest.TestCase): "version": 3, "containers": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "imageDetails": self.harness.charm.image.fetch(), "imagePullPolicy": "Always", "ports": [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "containerPort": 9308, "protocol": "TCP", } @@ -357,20 +357,20 @@ class TestCharm(unittest.TestCase): "kubernetesResources": { "ingressResources": [ { - "name": "prometheus-kafka-exporter-ingress", + "name": "kafka-exporter-ingress", "annotations": { "nginx.ingress.kubernetes.io/whitelist-source-range": "0.0.0.0/0", }, "spec": { "rules": [ { - "host": "prometheus-kafka-exporter", + "host": "kafka-exporter", "http": { "paths": [ { "path": "/", "backend": { - "serviceName": "prometheus-kafka-exporter", + "serviceName": "kafka-exporter", "servicePort": 9308, }, } @@ -380,8 +380,8 @@ class TestCharm(unittest.TestCase): ], "tls": [ { - "hosts": ["prometheus-kafka-exporter"], - "secretName": "prometheus-kafka-exporter", + "hosts": ["kafka-exporter"], + "secretName": "kafka-exporter", } ], }, @@ -406,8 +406,8 @@ class TestCharm(unittest.TestCase): self.harness.update_config( { - "site_url": "https://prometheus-kafka-exporter", - "tls_secret_name": "prometheus-kafka-exporter", + "site_url": "https://kafka-exporter", + "tls_secret_name": "kafka-exporter", "ingress_whitelist_source_range": "0.0.0.0/0", } ) @@ -437,17 +437,38 @@ class TestCharm(unittest.TestCase): def test_publish_target_info(self) -> NoReturn: """Test to see if target relation is updated.""" expected_result = { - "hostname": "prometheus-kafka-exporter", + "hostname": "kafka-exporter", "port": "9308", + "metrics_path": "/metrics", + "scrape_interval": "30s", + "scrape_timeout": "15s", } self.harness.charm.on.start.emit() - relation_id = self.harness.add_relation("prometheus-target", "prometheus") + relation_id = self.harness.add_relation("prometheus-scrape", "prometheus") self.harness.add_relation_unit(relation_id, "prometheus/0") - relation_data = self.harness.get_relation_data( - relation_id, "prometheus-kafka-exporter/0" - ) + relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0") + + self.assertDictEqual(expected_result, relation_data) + + def test_publish_target_info_with_site_url(self) -> NoReturn: + """Test to see if target relation is updated.""" + expected_result = { + "hostname": "kafka-exporter-osm", + "port": "80", + "metrics_path": "/metrics", + "scrape_interval": "30s", + "scrape_timeout": "15s", + } + + self.harness.charm.on.start.emit() + + self.harness.update_config({"site_url": "http://kafka-exporter-osm"}) + + relation_id = self.harness.add_relation("prometheus-scrape", "prometheus") + self.harness.add_relation_unit(relation_id, "prometheus/0") + relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0") self.assertDictEqual(expected_result, relation_data) @@ -457,9 +478,7 @@ class TestCharm(unittest.TestCase): relation_id = self.harness.add_relation("grafana-dashboard", "grafana") self.harness.add_relation_unit(relation_id, "grafana/0") - relation_data = self.harness.get_relation_data( - relation_id, "prometheus-kafka-exporter/0" - ) + relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0") self.assertTrue("dashboard" in relation_data) self.assertTrue(len(relation_data["dashboard"]) > 0) diff --git a/installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py b/installers/charm/kafka-exporter/tests/test_pod_spec.py similarity index 95% rename from installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py rename to installers/charm/kafka-exporter/tests/test_pod_spec.py index b1251f87..44d99d8e 100644 --- a/installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py +++ b/installers/charm/kafka-exporter/tests/test_pod_spec.py @@ -35,7 +35,7 @@ class TestPodSpec(unittest.TestCase): expected_result = [ { - "name": "prometheus-kafka-exporter", + "name": "kafka-exporter", "containerPort": port, "protocol": "TCP", } @@ -59,7 +59,7 @@ class TestPodSpec(unittest.TestCase): def test_make_pod_ingress_resources_without_site_url(self) -> NoReturn: """Testing make pod ingress resources without site_url.""" config = {"site_url": ""} - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 pod_ingress_resources = pod_spec._make_pod_ingress_resources( @@ -71,10 +71,10 @@ class TestPodSpec(unittest.TestCase): def test_make_pod_ingress_resources(self) -> NoReturn: """Testing make pod ingress resources.""" config = { - "site_url": "http://prometheus-kafka-exporter", + "site_url": "http://kafka-exporter", "ingress_whitelist_source_range": "", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = [ @@ -113,10 +113,10 @@ class TestPodSpec(unittest.TestCase): def test_make_pod_ingress_resources_with_whitelist_source_range(self) -> NoReturn: """Testing make pod ingress resources with whitelist_source_range.""" config = { - "site_url": "http://prometheus-kafka-exporter", + "site_url": "http://kafka-exporter", "ingress_whitelist_source_range": "0.0.0.0/0", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = [ @@ -158,12 +158,12 @@ class TestPodSpec(unittest.TestCase): def test_make_pod_ingress_resources_with_https(self) -> NoReturn: """Testing make pod ingress resources with HTTPs.""" config = { - "site_url": "https://prometheus-kafka-exporter", + "site_url": "https://kafka-exporter", "max_file_size": 0, "ingress_whitelist_source_range": "", "tls_secret_name": "", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = [ @@ -201,12 +201,12 @@ class TestPodSpec(unittest.TestCase): def test_make_pod_ingress_resources_with_https_tls_secret_name(self) -> NoReturn: """Testing make pod ingress resources with HTTPs and TLS secret name.""" config = { - "site_url": "https://prometheus-kafka-exporter", + "site_url": "https://kafka-exporter", "max_file_size": 0, "ingress_whitelist_source_range": "", "tls_secret_name": "secret_name", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = [ @@ -309,7 +309,7 @@ class TestPodSpec(unittest.TestCase): "kafka_host": "kafka", "kafka_port": "9090", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = { @@ -365,8 +365,8 @@ class TestPodSpec(unittest.TestCase): """Testing make pod spec.""" image_info = {"upstream-source": "bitnami/kafka-exporter:latest"} config = { - "site_url": "https://prometheus-kafka-exporter", - "tls_secret_name": "prometheus-kafka-exporter", + "site_url": "https://kafka-exporter", + "tls_secret_name": "kafka-exporter", "max_file_size": 0, "ingress_whitelist_source_range": "0.0.0.0/0", } @@ -374,7 +374,7 @@ class TestPodSpec(unittest.TestCase): "kafka_host": "kafka", "kafka_port": "9090", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 expected_result = { @@ -471,7 +471,7 @@ class TestPodSpec(unittest.TestCase): "kafka_host": "kafka", "kafka_port": "9090", } - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 spec = pod_spec.make_pod_spec( @@ -487,7 +487,7 @@ class TestPodSpec(unittest.TestCase): "site_url": "", } relation_state = {} - app_name = "prometheus-kafka-exporter" + app_name = "kafka-exporter" port = 9308 with self.assertRaises(ValueError): diff --git a/installers/charm/prometheus-kafka-exporter/tox.ini b/installers/charm/kafka-exporter/tox.ini similarity index 93% rename from installers/charm/prometheus-kafka-exporter/tox.ini rename to installers/charm/kafka-exporter/tox.ini index cf77aa6d..a6dfd317 100644 --- a/installers/charm/prometheus-kafka-exporter/tox.ini +++ b/installers/charm/kafka-exporter/tox.ini @@ -30,7 +30,7 @@ basepython = python3 setenv = PYTHONHASHSEED=0 PYTHONPATH = {toxinidir}/src - CHARM_NAME = prometheus-kafka-exporter + CHARM_NAME = kafka-exporter [testenv:build] passenv=HTTP_PROXY HTTPS_PROXY NO_PROXY @@ -39,9 +39,9 @@ whitelist_externals = rm unzip commands = - rm -rf release grafana.charm + rm -rf release kafka-exporter.charm charmcraft build - unzip prometheus-kafka-exporter.charm -d release + unzip kafka-exporter.charm -d release [testenv:unit] commands = diff --git a/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml b/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml deleted file mode 100644 index afcb7e3b..00000000 --- a/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml +++ /dev/null @@ -1,543 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: legal@canonical.com -# -# To get in touch with the maintainers, please contact: -# osm-charmers@lists.launchpad.net -## - ---- -annotations: - list: - - builtIn: 1 - datasource: '-- Grafana --' - enable: true - hide: true - iconColor: 'rgba(0, 211, 255, 1)' - name: Annotations & Alerts - type: dashboard -description: Kafka resource usage and throughput -editable: true -gnetId: 7589 -graphTooltip: 0 -id: 10 -iteration: 1578848023483 -links: [] -panels: - - aliasColors: {} - bars: false - dashLength: 10 - dashes: false - datasource: Prometheus - fill: 0 - fillGradient: 0 - gridPos: - h: 10 - w: 10 - x: 0 - 'y': 0 - id: 14 - legend: - alignAsTable: true - avg: false - current: true - max: true - min: false - rightSide: false - show: true - sideWidth: 480 - sort: max - sortDesc: true - total: false - values: true - lines: true - linewidth: 1 - links: [] - nullPointMode: connected - options: - dataLinks: [] - percentage: false - pointradius: 5 - points: false - renderer: flot - seriesOverrides: [] - spaceLength: 10 - stack: false - steppedLine: false - targets: - - expr: >- - sum(kafka_topic_partition_current_offset - - kafka_topic_partition_oldest_offset{instance="$instance", - topic=~"$topic"}) by (topic) - format: time_series - intervalFactor: 1 - legendFormat: '{{topic}}' - refId: B - thresholds: [] - timeFrom: null - timeRegions: [] - timeShift: null - title: Messages stored per topic - tooltip: - shared: true - sort: 0 - value_type: individual - type: graph - xaxis: - buckets: null - mode: time - name: null - show: true - values: [] - yaxes: - - format: short - label: null - logBase: 1 - max: null - min: '0' - show: true - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - yaxis: - align: false - alignLevel: null - - aliasColors: {} - bars: false - dashLength: 10 - dashes: false - datasource: Prometheus - fill: 0 - fillGradient: 0 - gridPos: - h: 10 - w: 10 - x: 10 - 'y': 0 - id: 12 - legend: - alignAsTable: true - avg: false - current: true - max: true - min: false - rightSide: false - show: true - sideWidth: 480 - sort: max - sortDesc: true - total: false - values: true - lines: true - linewidth: 1 - links: [] - nullPointMode: connected - options: - dataLinks: [] - percentage: false - pointradius: 5 - points: false - renderer: flot - seriesOverrides: [] - spaceLength: 10 - stack: false - steppedLine: false - targets: - - expr: >- - sum(kafka_consumergroup_lag{instance="$instance",topic=~"$topic"}) by - (consumergroup, topic) - format: time_series - instant: false - interval: '' - intervalFactor: 1 - legendFormat: ' {{topic}} ({{consumergroup}})' - refId: A - thresholds: [] - timeFrom: null - timeRegions: [] - timeShift: null - title: Lag by Consumer Group - tooltip: - shared: true - sort: 2 - value_type: individual - type: graph - xaxis: - buckets: null - mode: time - name: null - show: true - values: [] - yaxes: - - format: short - label: '' - logBase: 1 - max: null - min: '0' - show: true - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - yaxis: - align: false - alignLevel: null - - aliasColors: {} - bars: false - dashLength: 10 - dashes: false - datasource: Prometheus - fill: 0 - fillGradient: 0 - gridPos: - h: 10 - w: 10 - x: 0 - 'y': 10 - id: 16 - legend: - alignAsTable: true - avg: false - current: true - max: true - min: false - rightSide: false - show: true - sideWidth: 480 - total: false - values: true - lines: true - linewidth: 1 - links: [] - nullPointMode: connected - options: - dataLinks: [] - percentage: false - pointradius: 5 - points: false - renderer: flot - seriesOverrides: [] - spaceLength: 10 - stack: false - steppedLine: false - targets: - - expr: >- - sum(delta(kafka_topic_partition_current_offset{instance=~'$instance', - topic=~"$topic"}[5m])/5) by (topic) - format: time_series - intervalFactor: 1 - legendFormat: '{{topic}}' - refId: A - thresholds: [] - timeFrom: null - timeRegions: [] - timeShift: null - title: Messages produced per minute - tooltip: - shared: true - sort: 0 - value_type: individual - type: graph - xaxis: - buckets: null - mode: time - name: null - show: true - values: [] - yaxes: - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - yaxis: - align: false - alignLevel: null - - aliasColors: {} - bars: false - dashLength: 10 - dashes: false - datasource: Prometheus - fill: 0 - fillGradient: 0 - gridPos: - h: 10 - w: 10 - x: 10 - 'y': 10 - id: 18 - legend: - alignAsTable: true - avg: false - current: true - max: true - min: false - rightSide: false - show: true - sideWidth: 480 - sort: current - sortDesc: true - total: false - values: true - lines: true - linewidth: 1 - links: [] - nullPointMode: connected - options: - dataLinks: [] - percentage: false - pointradius: 5 - points: false - renderer: flot - seriesOverrides: [] - spaceLength: 10 - stack: false - steppedLine: false - targets: - - expr: >- - sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~"$topic"}[5m])/5) - by (consumergroup, topic) - format: time_series - intervalFactor: 1 - legendFormat: ' {{topic}} ({{consumergroup}})' - refId: A - thresholds: [] - timeFrom: null - timeRegions: [] - timeShift: null - title: Messages consumed per minute - tooltip: - shared: true - sort: 0 - value_type: individual - type: graph - xaxis: - buckets: null - mode: time - name: null - show: true - values: [] - yaxes: - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - yaxis: - align: false - alignLevel: null - - aliasColors: {} - bars: true - dashLength: 10 - dashes: false - datasource: Prometheus - fill: 1 - fillGradient: 0 - gridPos: - h: 7 - w: 20 - x: 0 - 'y': 20 - id: 8 - legend: - alignAsTable: true - avg: false - current: true - max: false - min: false - rightSide: true - show: true - sideWidth: 420 - total: false - values: true - lines: false - linewidth: 1 - links: [] - nullPointMode: 'null' - options: - dataLinks: [] - percentage: false - pointradius: 5 - points: false - renderer: flot - seriesOverrides: [] - spaceLength: 10 - stack: false - steppedLine: false - targets: - - expr: >- - sum by(topic) - (kafka_topic_partitions{instance="$instance",topic=~"$topic"}) - format: time_series - intervalFactor: 1 - legendFormat: '{{topic}}' - refId: A - thresholds: [] - timeFrom: null - timeRegions: [] - timeShift: null - title: Partitions per Topic - tooltip: - shared: false - sort: 0 - value_type: individual - type: graph - xaxis: - buckets: null - mode: series - name: null - show: false - values: - - current - yaxes: - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - - format: short - label: null - logBase: 1 - max: null - min: null - show: true - yaxis: - align: false - alignLevel: null -refresh: 5s -schemaVersion: 19 -style: dark -tags: [] -templating: - list: - - allValue: null - current: - text: osm-kafka-exporter-service - value: osm-kafka-exporter-service - datasource: Prometheus - definition: '' - hide: 0 - includeAll: false - label: Job - multi: false - name: job - options: [] - query: 'label_values(kafka_consumergroup_current_offset, job)' - refresh: 1 - regex: '' - skipUrlSync: false - sort: 0 - tagValuesQuery: '' - tags: [] - tagsQuery: '' - type: query - useTags: false - - allValue: null - datasource: Prometheus - definition: '' - hide: 0 - includeAll: false - label: Instance - multi: false - name: instance - options: [] - query: >- - 'label_values(kafka_consumergroup_current_offset{job=~"$job"},' - 'instance)' - refresh: 1 - regex: '' - skipUrlSync: false - sort: 0 - tagValuesQuery: '' - tags: [] - tagsQuery: '' - type: query - useTags: false - - allValue: null - current: - tags: [] - text: All - value: - - $__all - datasource: Prometheus - definition: '' - hide: 0 - includeAll: true - label: Topic - multi: true - name: topic - options: [] - query: >- - label_values(kafka_topic_partition_current_offset{instance='$instance',topic!='__consumer_offsets',topic!='--kafka'}, - topic) - refresh: 1 - regex: '' - skipUrlSync: false - sort: 1 - tagValuesQuery: '' - tags: [] - tagsQuery: topic - type: query - useTags: false -time: - from: now-1h - to: now -timepicker: - refresh_intervals: - - 5s - - 10s - - 30s - - 1m - - 5m - - 15m - - 30m - - 1h - - 2h - - 1d - time_options: - - 5m - - 15m - - 1h - - 6h - - 12h - - 24h - - 2d - - 7d - - 30d -timezone: browser -title: Kafka -uid: jwPKIsniz -version: 2 -- 2.25.1