Adding scraping relation to Kafka Exporter
Change-Id: Iea3c8f39f8a90fb4e287b074bf06f1042b71857c
Signed-off-by: sousaedu <eduardo.sousa@canonical.com>
diff --git a/installers/charm/prometheus-kafka-exporter/.gitignore b/installers/charm/kafka-exporter/.gitignore
similarity index 95%
rename from installers/charm/prometheus-kafka-exporter/.gitignore
rename to installers/charm/kafka-exporter/.gitignore
index d1c8218..0933edc 100644
--- a/installers/charm/prometheus-kafka-exporter/.gitignore
+++ b/installers/charm/kafka-exporter/.gitignore
@@ -22,7 +22,7 @@
venv
.vscode
build
-prometheus-kafka-exporter.charm
+kafka-exporter.charm
.coverage
.stestr
cover
diff --git a/installers/charm/prometheus-kafka-exporter/.yamllint.yaml b/installers/charm/kafka-exporter/.yamllint.yaml
similarity index 100%
rename from installers/charm/prometheus-kafka-exporter/.yamllint.yaml
rename to installers/charm/kafka-exporter/.yamllint.yaml
diff --git a/installers/charm/prometheus-kafka-exporter/README.md b/installers/charm/kafka-exporter/README.md
similarity index 100%
rename from installers/charm/prometheus-kafka-exporter/README.md
rename to installers/charm/kafka-exporter/README.md
diff --git a/installers/charm/prometheus-kafka-exporter/config.yaml b/installers/charm/kafka-exporter/config.yaml
similarity index 100%
rename from installers/charm/prometheus-kafka-exporter/config.yaml
rename to installers/charm/kafka-exporter/config.yaml
diff --git a/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json b/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json
new file mode 100644
index 0000000..5b7552a
--- /dev/null
+++ b/installers/charm/kafka-exporter/files/kafka_exporter_dashboard.json
@@ -0,0 +1,609 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Kafka resource usage and throughput",
+ "editable": true,
+ "gnetId": 7589,
+ "graphTooltip": 0,
+ "id": 10,
+ "iteration": 1578848023483,
+ "links": [],
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus - Juju generated source",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 10,
+ "x": 0,
+ "y": 0
+ },
+ "id": 14,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 480,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kafka_topic_partition_current_offset - kafka_topic_partition_oldest_offset{instance=\"$instance\", topic=~\"$topic\"}) by (topic)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Messages stored per topic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus - Juju generated source",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 10,
+ "x": 10,
+ "y": 0
+ },
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 480,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kafka_consumergroup_lag{instance=\"$instance\",topic=~\"$topic\"}) by (consumergroup, topic) ",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": " {{topic}} ({{consumergroup}})",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Lag by Consumer Group",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus - Juju generated source",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 10,
+ "x": 0,
+ "y": 10
+ },
+ "id": 16,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 480,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(delta(kafka_topic_partition_current_offset{instance=~'$instance', topic=~\"$topic\"}[5m])/5) by (topic)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Messages produced per minute",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus - Juju generated source",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 10,
+ "x": 10,
+ "y": 10
+ },
+ "id": 18,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 480,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[5m])/5) by (consumergroup, topic)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": " {{topic}} ({{consumergroup}})",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Messages consumed per minute",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus - Juju generated source",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 20,
+ "x": 0,
+ "y": 20
+ },
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 420,
+ "total": false,
+ "values": true
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(topic) (kafka_topic_partitions{instance=\"$instance\",topic=~\"$topic\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Partitions per Topic",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "series",
+ "name": null,
+ "show": false,
+ "values": [
+ "current"
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "5s",
+ "schemaVersion": 19,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {
+ "text": "osm-kafka-exporter-service",
+ "value": "osm-kafka-exporter-service"
+ },
+ "datasource": "prometheus - Juju generated source",
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Job",
+ "multi": false,
+ "name": "job",
+ "options": [],
+ "query": "label_values(kafka_consumergroup_current_offset, job)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "datasource": "prometheus - Juju generated source",
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Instance",
+ "multi": false,
+ "name": "instance",
+ "options": [],
+ "query": "label_values(kafka_consumergroup_current_offset{job=~\"$job\"}, instance)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "tags": [],
+ "text": "All",
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": "prometheus - Juju generated source",
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Topic",
+ "multi": true,
+ "name": "topic",
+ "options": [],
+ "query": "label_values(kafka_topic_partition_current_offset{instance='$instance',topic!='__consumer_offsets',topic!='--kafka'}, topic)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "topic",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Kafka",
+ "uid": "jwPKIsniz",
+ "version": 2
+}
diff --git a/installers/charm/prometheus-kafka-exporter/metadata.yaml b/installers/charm/kafka-exporter/metadata.yaml
similarity index 94%
rename from installers/charm/prometheus-kafka-exporter/metadata.yaml
rename to installers/charm/kafka-exporter/metadata.yaml
index 55f2ab0..1339d27 100644
--- a/installers/charm/prometheus-kafka-exporter/metadata.yaml
+++ b/installers/charm/kafka-exporter/metadata.yaml
@@ -19,7 +19,7 @@
# osm-charmers@lists.launchpad.net
##
-name: prometheus-kafka-exporter
+name: kafka-exporter
summary: OSM Prometheus Kafka Exporter
description: |
A CAAS charm to deploy OSM's Prometheus Kafka Exporter.
@@ -43,7 +43,7 @@
kafka:
interface: kafka
provides:
- prometheus-target:
- interface: http
+ prometheus-scrape:
+ interface: prometheus
grafana-dashboard:
interface: grafana-dashboard
diff --git a/installers/charm/prometheus-kafka-exporter/requirements.txt b/installers/charm/kafka-exporter/requirements.txt
similarity index 100%
rename from installers/charm/prometheus-kafka-exporter/requirements.txt
rename to installers/charm/kafka-exporter/requirements.txt
diff --git a/installers/charm/prometheus-kafka-exporter/src/charm.py b/installers/charm/kafka-exporter/src/charm.py
similarity index 87%
rename from installers/charm/prometheus-kafka-exporter/src/charm.py
rename to installers/charm/kafka-exporter/src/charm.py
index 70a31f8..9f03a34 100755
--- a/installers/charm/prometheus-kafka-exporter/src/charm.py
+++ b/installers/charm/kafka-exporter/src/charm.py
@@ -23,6 +23,7 @@
import logging
from pathlib import Path
from typing import Dict, List, NoReturn
+from urllib.parse import urlparse
from ops.charm import CharmBase
from ops.framework import EventBase, StoredState
@@ -34,7 +35,7 @@
logger = logging.getLogger(__name__)
-PROMETHEUS_KAFKA_EXPORTER_PORT = 9308
+KAFKA_EXPORTER_PORT = 9308
class RelationsMissing(Exception):
@@ -93,19 +94,19 @@
return data
-class PrometheusKafkaExporterCharm(CharmBase):
- """Prometheus Kafka Exporter Charm."""
+class KafkaExporterCharm(CharmBase):
+ """Kafka Exporter Charm."""
state = StoredState()
def __init__(self, *args) -> NoReturn:
- """Prometheus Kafka Exporter Charm constructor."""
+ """Kafka Exporter Charm constructor."""
super().__init__(*args)
# Internal state initialization
self.state.set_default(pod_spec=None)
- self.port = PROMETHEUS_KAFKA_EXPORTER_PORT
+ self.port = KAFKA_EXPORTER_PORT
self.image = OCIImageResource(self, "image")
# Registering regular events
@@ -120,21 +121,26 @@
# Registering provided relation events
self.framework.observe(
- self.on.prometheus_target_relation_joined, self._publish_target_info
+ self.on.prometheus_scrape_relation_joined, self._publish_scrape_info
)
self.framework.observe(
self.on.grafana_dashboard_relation_joined, self._publish_dashboard_info
)
- def _publish_target_info(self, event: EventBase) -> NoReturn:
- """Publishes target information.
+ def _publish_scrape_info(self, event: EventBase) -> NoReturn:
+ """Publishes scrape information.
Args:
event (EventBase): Exporter relation event.
"""
rel_data = {
- "hostname": self.model.app.name,
- "port": str(PROMETHEUS_KAFKA_EXPORTER_PORT),
+ "hostname": urlparse(self.model.config["site_url"]).hostname
+ if self.model.config["site_url"]
+ else self.model.app.name,
+ "port": "80" if self.model.config["site_url"] else str(KAFKA_EXPORTER_PORT),
+ "metrics_path": "/metrics",
+ "scrape_interval": "30s",
+ "scrape_timeout": "15s",
}
for k, v in rel_data.items():
event.relation.data[self.unit][k] = v
@@ -146,7 +152,8 @@
event (EventBase): Exporter relation event.
"""
rel_data = {
- "dashboard": Path("files/kafka_exporter_dashboard.yaml").read_text(),
+ "name": "osm-kafka",
+ "dashboard": Path("files/kafka_exporter_dashboard.json").read_text(),
}
for k, v in rel_data.items():
event.relation.data[self.unit][k] = v
@@ -213,4 +220,4 @@
if __name__ == "__main__":
- main(PrometheusKafkaExporterCharm)
+ main(KafkaExporterCharm)
diff --git a/installers/charm/prometheus-kafka-exporter/src/pod_spec.py b/installers/charm/kafka-exporter/src/pod_spec.py
similarity index 97%
rename from installers/charm/prometheus-kafka-exporter/src/pod_spec.py
rename to installers/charm/kafka-exporter/src/pod_spec.py
index 256c203..a50d96f 100644
--- a/installers/charm/prometheus-kafka-exporter/src/pod_spec.py
+++ b/installers/charm/kafka-exporter/src/pod_spec.py
@@ -102,9 +102,7 @@
Returns:
List[Dict[str, Any]]: pod port details.
"""
- return [
- {"name": "prometheus-kafka-exporter", "containerPort": port, "protocol": "TCP"}
- ]
+ return [{"name": "kafka-exporter", "containerPort": port, "protocol": "TCP"}]
def _make_pod_envconfig(
@@ -246,7 +244,7 @@
List[str]: command to startup the process.
"""
command = [
- "kafka-exporter",
+ "kafka_exporter",
"--kafka.server={}:{}".format(
relation.get("kafka_host"), relation.get("kafka_port")
),
@@ -259,7 +257,7 @@
image_info: Dict[str, str],
config: Dict[str, Any],
relation_state: Dict[str, Any],
- app_name: str = "prometheus-kafka-exporter",
+ app_name: str = "kafka-exporter",
port: int = 9308,
) -> Dict[str, Any]:
"""Generate the pod spec information.
diff --git a/installers/charm/prometheus-kafka-exporter/tests/__init__.py b/installers/charm/kafka-exporter/tests/__init__.py
similarity index 100%
rename from installers/charm/prometheus-kafka-exporter/tests/__init__.py
rename to installers/charm/kafka-exporter/tests/__init__.py
diff --git a/installers/charm/prometheus-kafka-exporter/tests/test_charm.py b/installers/charm/kafka-exporter/tests/test_charm.py
similarity index 85%
rename from installers/charm/prometheus-kafka-exporter/tests/test_charm.py
rename to installers/charm/kafka-exporter/tests/test_charm.py
index 6b50874..5361321 100644
--- a/installers/charm/prometheus-kafka-exporter/tests/test_charm.py
+++ b/installers/charm/kafka-exporter/tests/test_charm.py
@@ -26,15 +26,15 @@
from ops.model import BlockedStatus
from ops.testing import Harness
-from charm import PrometheusKafkaExporterCharm
+from charm import KafkaExporterCharm
class TestCharm(unittest.TestCase):
- """Prometheus Kafka Exporter Charm unit tests."""
+ """Kafka Exporter Charm unit tests."""
def setUp(self) -> NoReturn:
"""Test setup"""
- self.harness = Harness(PrometheusKafkaExporterCharm)
+ self.harness = Harness(KafkaExporterCharm)
self.harness.set_leader(is_leader=True)
self.harness.begin()
@@ -59,12 +59,12 @@
"version": 3,
"containers": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"imageDetails": self.harness.charm.image.fetch(),
"imagePullPolicy": "Always",
"ports": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"containerPort": 9308,
"protocol": "TCP",
}
@@ -125,12 +125,12 @@
"version": 3,
"containers": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"imageDetails": self.harness.charm.image.fetch(),
"imagePullPolicy": "Always",
"ports": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"containerPort": 9308,
"protocol": "TCP",
}
@@ -164,20 +164,20 @@
"kubernetesResources": {
"ingressResources": [
{
- "name": "prometheus-kafka-exporter-ingress",
+ "name": "kafka-exporter-ingress",
"annotations": {
"nginx.ingress.kubernetes.io/ssl-redirect": "false",
},
"spec": {
"rules": [
{
- "host": "prometheus-kafka-exporter",
+ "host": "kafka-exporter",
"http": {
"paths": [
{
"path": "/",
"backend": {
- "serviceName": "prometheus-kafka-exporter",
+ "serviceName": "kafka-exporter",
"servicePort": 9308,
},
}
@@ -205,7 +205,7 @@
},
)
- self.harness.update_config({"site_url": "http://prometheus-kafka-exporter"})
+ self.harness.update_config({"site_url": "http://kafka-exporter"})
pod_spec, _ = self.harness.get_pod_spec()
@@ -217,12 +217,12 @@
"version": 3,
"containers": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"imageDetails": self.harness.charm.image.fetch(),
"imagePullPolicy": "Always",
"ports": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"containerPort": 9308,
"protocol": "TCP",
}
@@ -256,18 +256,18 @@
"kubernetesResources": {
"ingressResources": [
{
- "name": "prometheus-kafka-exporter-ingress",
+ "name": "kafka-exporter-ingress",
"annotations": {},
"spec": {
"rules": [
{
- "host": "prometheus-kafka-exporter",
+ "host": "kafka-exporter",
"http": {
"paths": [
{
"path": "/",
"backend": {
- "serviceName": "prometheus-kafka-exporter",
+ "serviceName": "kafka-exporter",
"servicePort": 9308,
},
}
@@ -277,8 +277,8 @@
],
"tls": [
{
- "hosts": ["prometheus-kafka-exporter"],
- "secretName": "prometheus-kafka-exporter",
+ "hosts": ["kafka-exporter"],
+ "secretName": "kafka-exporter",
}
],
},
@@ -303,8 +303,8 @@
self.harness.update_config(
{
- "site_url": "https://prometheus-kafka-exporter",
- "tls_secret_name": "prometheus-kafka-exporter",
+ "site_url": "https://kafka-exporter",
+ "tls_secret_name": "kafka-exporter",
}
)
@@ -318,12 +318,12 @@
"version": 3,
"containers": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"imageDetails": self.harness.charm.image.fetch(),
"imagePullPolicy": "Always",
"ports": [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"containerPort": 9308,
"protocol": "TCP",
}
@@ -357,20 +357,20 @@
"kubernetesResources": {
"ingressResources": [
{
- "name": "prometheus-kafka-exporter-ingress",
+ "name": "kafka-exporter-ingress",
"annotations": {
"nginx.ingress.kubernetes.io/whitelist-source-range": "0.0.0.0/0",
},
"spec": {
"rules": [
{
- "host": "prometheus-kafka-exporter",
+ "host": "kafka-exporter",
"http": {
"paths": [
{
"path": "/",
"backend": {
- "serviceName": "prometheus-kafka-exporter",
+ "serviceName": "kafka-exporter",
"servicePort": 9308,
},
}
@@ -380,8 +380,8 @@
],
"tls": [
{
- "hosts": ["prometheus-kafka-exporter"],
- "secretName": "prometheus-kafka-exporter",
+ "hosts": ["kafka-exporter"],
+ "secretName": "kafka-exporter",
}
],
},
@@ -406,8 +406,8 @@
self.harness.update_config(
{
- "site_url": "https://prometheus-kafka-exporter",
- "tls_secret_name": "prometheus-kafka-exporter",
+ "site_url": "https://kafka-exporter",
+ "tls_secret_name": "kafka-exporter",
"ingress_whitelist_source_range": "0.0.0.0/0",
}
)
@@ -437,17 +437,38 @@
def test_publish_target_info(self) -> NoReturn:
"""Test to see if target relation is updated."""
expected_result = {
- "hostname": "prometheus-kafka-exporter",
+ "hostname": "kafka-exporter",
"port": "9308",
+ "metrics_path": "/metrics",
+ "scrape_interval": "30s",
+ "scrape_timeout": "15s",
}
self.harness.charm.on.start.emit()
- relation_id = self.harness.add_relation("prometheus-target", "prometheus")
+ relation_id = self.harness.add_relation("prometheus-scrape", "prometheus")
self.harness.add_relation_unit(relation_id, "prometheus/0")
- relation_data = self.harness.get_relation_data(
- relation_id, "prometheus-kafka-exporter/0"
- )
+ relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0")
+
+ self.assertDictEqual(expected_result, relation_data)
+
+ def test_publish_target_info_with_site_url(self) -> NoReturn:
+ """Test to see if target relation is updated."""
+ expected_result = {
+ "hostname": "kafka-exporter-osm",
+ "port": "80",
+ "metrics_path": "/metrics",
+ "scrape_interval": "30s",
+ "scrape_timeout": "15s",
+ }
+
+ self.harness.charm.on.start.emit()
+
+ self.harness.update_config({"site_url": "http://kafka-exporter-osm"})
+
+ relation_id = self.harness.add_relation("prometheus-scrape", "prometheus")
+ self.harness.add_relation_unit(relation_id, "prometheus/0")
+ relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0")
self.assertDictEqual(expected_result, relation_data)
@@ -457,9 +478,7 @@
relation_id = self.harness.add_relation("grafana-dashboard", "grafana")
self.harness.add_relation_unit(relation_id, "grafana/0")
- relation_data = self.harness.get_relation_data(
- relation_id, "prometheus-kafka-exporter/0"
- )
+ relation_data = self.harness.get_relation_data(relation_id, "kafka-exporter/0")
self.assertTrue("dashboard" in relation_data)
self.assertTrue(len(relation_data["dashboard"]) > 0)
diff --git a/installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py b/installers/charm/kafka-exporter/tests/test_pod_spec.py
similarity index 94%
rename from installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py
rename to installers/charm/kafka-exporter/tests/test_pod_spec.py
index b1251f8..44d99d8 100644
--- a/installers/charm/prometheus-kafka-exporter/tests/test_pod_spec.py
+++ b/installers/charm/kafka-exporter/tests/test_pod_spec.py
@@ -35,7 +35,7 @@
expected_result = [
{
- "name": "prometheus-kafka-exporter",
+ "name": "kafka-exporter",
"containerPort": port,
"protocol": "TCP",
}
@@ -59,7 +59,7 @@
def test_make_pod_ingress_resources_without_site_url(self) -> NoReturn:
"""Testing make pod ingress resources without site_url."""
config = {"site_url": ""}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
pod_ingress_resources = pod_spec._make_pod_ingress_resources(
@@ -71,10 +71,10 @@
def test_make_pod_ingress_resources(self) -> NoReturn:
"""Testing make pod ingress resources."""
config = {
- "site_url": "http://prometheus-kafka-exporter",
+ "site_url": "http://kafka-exporter",
"ingress_whitelist_source_range": "",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = [
@@ -113,10 +113,10 @@
def test_make_pod_ingress_resources_with_whitelist_source_range(self) -> NoReturn:
"""Testing make pod ingress resources with whitelist_source_range."""
config = {
- "site_url": "http://prometheus-kafka-exporter",
+ "site_url": "http://kafka-exporter",
"ingress_whitelist_source_range": "0.0.0.0/0",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = [
@@ -158,12 +158,12 @@
def test_make_pod_ingress_resources_with_https(self) -> NoReturn:
"""Testing make pod ingress resources with HTTPs."""
config = {
- "site_url": "https://prometheus-kafka-exporter",
+ "site_url": "https://kafka-exporter",
"max_file_size": 0,
"ingress_whitelist_source_range": "",
"tls_secret_name": "",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = [
@@ -201,12 +201,12 @@
def test_make_pod_ingress_resources_with_https_tls_secret_name(self) -> NoReturn:
"""Testing make pod ingress resources with HTTPs and TLS secret name."""
config = {
- "site_url": "https://prometheus-kafka-exporter",
+ "site_url": "https://kafka-exporter",
"max_file_size": 0,
"ingress_whitelist_source_range": "",
"tls_secret_name": "secret_name",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = [
@@ -309,7 +309,7 @@
"kafka_host": "kafka",
"kafka_port": "9090",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = {
@@ -365,8 +365,8 @@
"""Testing make pod spec."""
image_info = {"upstream-source": "bitnami/kafka-exporter:latest"}
config = {
- "site_url": "https://prometheus-kafka-exporter",
- "tls_secret_name": "prometheus-kafka-exporter",
+ "site_url": "https://kafka-exporter",
+ "tls_secret_name": "kafka-exporter",
"max_file_size": 0,
"ingress_whitelist_source_range": "0.0.0.0/0",
}
@@ -374,7 +374,7 @@
"kafka_host": "kafka",
"kafka_port": "9090",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
expected_result = {
@@ -471,7 +471,7 @@
"kafka_host": "kafka",
"kafka_port": "9090",
}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
spec = pod_spec.make_pod_spec(
@@ -487,7 +487,7 @@
"site_url": "",
}
relation_state = {}
- app_name = "prometheus-kafka-exporter"
+ app_name = "kafka-exporter"
port = 9308
with self.assertRaises(ValueError):
diff --git a/installers/charm/prometheus-kafka-exporter/tox.ini b/installers/charm/kafka-exporter/tox.ini
similarity index 93%
rename from installers/charm/prometheus-kafka-exporter/tox.ini
rename to installers/charm/kafka-exporter/tox.ini
index cf77aa6..a6dfd31 100644
--- a/installers/charm/prometheus-kafka-exporter/tox.ini
+++ b/installers/charm/kafka-exporter/tox.ini
@@ -30,7 +30,7 @@
setenv =
PYTHONHASHSEED=0
PYTHONPATH = {toxinidir}/src
- CHARM_NAME = prometheus-kafka-exporter
+ CHARM_NAME = kafka-exporter
[testenv:build]
passenv=HTTP_PROXY HTTPS_PROXY NO_PROXY
@@ -39,9 +39,9 @@
rm
unzip
commands =
- rm -rf release grafana.charm
+ rm -rf release kafka-exporter.charm
charmcraft build
- unzip prometheus-kafka-exporter.charm -d release
+ unzip kafka-exporter.charm -d release
[testenv:unit]
commands =
diff --git a/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml b/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml
deleted file mode 100644
index afcb7e3..0000000
--- a/installers/charm/prometheus-kafka-exporter/files/kafka_exporter_dashboard.yaml
+++ /dev/null
@@ -1,543 +0,0 @@
-# Copyright 2021 Canonical Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: legal@canonical.com
-#
-# To get in touch with the maintainers, please contact:
-# osm-charmers@lists.launchpad.net
-##
-
----
-annotations:
- list:
- - builtIn: 1
- datasource: '-- Grafana --'
- enable: true
- hide: true
- iconColor: 'rgba(0, 211, 255, 1)'
- name: Annotations & Alerts
- type: dashboard
-description: Kafka resource usage and throughput
-editable: true
-gnetId: 7589
-graphTooltip: 0
-id: 10
-iteration: 1578848023483
-links: []
-panels:
- - aliasColors: {}
- bars: false
- dashLength: 10
- dashes: false
- datasource: Prometheus
- fill: 0
- fillGradient: 0
- gridPos:
- h: 10
- w: 10
- x: 0
- 'y': 0
- id: 14
- legend:
- alignAsTable: true
- avg: false
- current: true
- max: true
- min: false
- rightSide: false
- show: true
- sideWidth: 480
- sort: max
- sortDesc: true
- total: false
- values: true
- lines: true
- linewidth: 1
- links: []
- nullPointMode: connected
- options:
- dataLinks: []
- percentage: false
- pointradius: 5
- points: false
- renderer: flot
- seriesOverrides: []
- spaceLength: 10
- stack: false
- steppedLine: false
- targets:
- - expr: >-
- sum(kafka_topic_partition_current_offset -
- kafka_topic_partition_oldest_offset{instance="$instance",
- topic=~"$topic"}) by (topic)
- format: time_series
- intervalFactor: 1
- legendFormat: '{{topic}}'
- refId: B
- thresholds: []
- timeFrom: null
- timeRegions: []
- timeShift: null
- title: Messages stored per topic
- tooltip:
- shared: true
- sort: 0
- value_type: individual
- type: graph
- xaxis:
- buckets: null
- mode: time
- name: null
- show: true
- values: []
- yaxes:
- - format: short
- label: null
- logBase: 1
- max: null
- min: '0'
- show: true
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- yaxis:
- align: false
- alignLevel: null
- - aliasColors: {}
- bars: false
- dashLength: 10
- dashes: false
- datasource: Prometheus
- fill: 0
- fillGradient: 0
- gridPos:
- h: 10
- w: 10
- x: 10
- 'y': 0
- id: 12
- legend:
- alignAsTable: true
- avg: false
- current: true
- max: true
- min: false
- rightSide: false
- show: true
- sideWidth: 480
- sort: max
- sortDesc: true
- total: false
- values: true
- lines: true
- linewidth: 1
- links: []
- nullPointMode: connected
- options:
- dataLinks: []
- percentage: false
- pointradius: 5
- points: false
- renderer: flot
- seriesOverrides: []
- spaceLength: 10
- stack: false
- steppedLine: false
- targets:
- - expr: >-
- sum(kafka_consumergroup_lag{instance="$instance",topic=~"$topic"}) by
- (consumergroup, topic)
- format: time_series
- instant: false
- interval: ''
- intervalFactor: 1
- legendFormat: ' {{topic}} ({{consumergroup}})'
- refId: A
- thresholds: []
- timeFrom: null
- timeRegions: []
- timeShift: null
- title: Lag by Consumer Group
- tooltip:
- shared: true
- sort: 2
- value_type: individual
- type: graph
- xaxis:
- buckets: null
- mode: time
- name: null
- show: true
- values: []
- yaxes:
- - format: short
- label: ''
- logBase: 1
- max: null
- min: '0'
- show: true
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- yaxis:
- align: false
- alignLevel: null
- - aliasColors: {}
- bars: false
- dashLength: 10
- dashes: false
- datasource: Prometheus
- fill: 0
- fillGradient: 0
- gridPos:
- h: 10
- w: 10
- x: 0
- 'y': 10
- id: 16
- legend:
- alignAsTable: true
- avg: false
- current: true
- max: true
- min: false
- rightSide: false
- show: true
- sideWidth: 480
- total: false
- values: true
- lines: true
- linewidth: 1
- links: []
- nullPointMode: connected
- options:
- dataLinks: []
- percentage: false
- pointradius: 5
- points: false
- renderer: flot
- seriesOverrides: []
- spaceLength: 10
- stack: false
- steppedLine: false
- targets:
- - expr: >-
- sum(delta(kafka_topic_partition_current_offset{instance=~'$instance',
- topic=~"$topic"}[5m])/5) by (topic)
- format: time_series
- intervalFactor: 1
- legendFormat: '{{topic}}'
- refId: A
- thresholds: []
- timeFrom: null
- timeRegions: []
- timeShift: null
- title: Messages produced per minute
- tooltip:
- shared: true
- sort: 0
- value_type: individual
- type: graph
- xaxis:
- buckets: null
- mode: time
- name: null
- show: true
- values: []
- yaxes:
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- yaxis:
- align: false
- alignLevel: null
- - aliasColors: {}
- bars: false
- dashLength: 10
- dashes: false
- datasource: Prometheus
- fill: 0
- fillGradient: 0
- gridPos:
- h: 10
- w: 10
- x: 10
- 'y': 10
- id: 18
- legend:
- alignAsTable: true
- avg: false
- current: true
- max: true
- min: false
- rightSide: false
- show: true
- sideWidth: 480
- sort: current
- sortDesc: true
- total: false
- values: true
- lines: true
- linewidth: 1
- links: []
- nullPointMode: connected
- options:
- dataLinks: []
- percentage: false
- pointradius: 5
- points: false
- renderer: flot
- seriesOverrides: []
- spaceLength: 10
- stack: false
- steppedLine: false
- targets:
- - expr: >-
- sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~"$topic"}[5m])/5)
- by (consumergroup, topic)
- format: time_series
- intervalFactor: 1
- legendFormat: ' {{topic}} ({{consumergroup}})'
- refId: A
- thresholds: []
- timeFrom: null
- timeRegions: []
- timeShift: null
- title: Messages consumed per minute
- tooltip:
- shared: true
- sort: 0
- value_type: individual
- type: graph
- xaxis:
- buckets: null
- mode: time
- name: null
- show: true
- values: []
- yaxes:
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- yaxis:
- align: false
- alignLevel: null
- - aliasColors: {}
- bars: true
- dashLength: 10
- dashes: false
- datasource: Prometheus
- fill: 1
- fillGradient: 0
- gridPos:
- h: 7
- w: 20
- x: 0
- 'y': 20
- id: 8
- legend:
- alignAsTable: true
- avg: false
- current: true
- max: false
- min: false
- rightSide: true
- show: true
- sideWidth: 420
- total: false
- values: true
- lines: false
- linewidth: 1
- links: []
- nullPointMode: 'null'
- options:
- dataLinks: []
- percentage: false
- pointradius: 5
- points: false
- renderer: flot
- seriesOverrides: []
- spaceLength: 10
- stack: false
- steppedLine: false
- targets:
- - expr: >-
- sum by(topic)
- (kafka_topic_partitions{instance="$instance",topic=~"$topic"})
- format: time_series
- intervalFactor: 1
- legendFormat: '{{topic}}'
- refId: A
- thresholds: []
- timeFrom: null
- timeRegions: []
- timeShift: null
- title: Partitions per Topic
- tooltip:
- shared: false
- sort: 0
- value_type: individual
- type: graph
- xaxis:
- buckets: null
- mode: series
- name: null
- show: false
- values:
- - current
- yaxes:
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- - format: short
- label: null
- logBase: 1
- max: null
- min: null
- show: true
- yaxis:
- align: false
- alignLevel: null
-refresh: 5s
-schemaVersion: 19
-style: dark
-tags: []
-templating:
- list:
- - allValue: null
- current:
- text: osm-kafka-exporter-service
- value: osm-kafka-exporter-service
- datasource: Prometheus
- definition: ''
- hide: 0
- includeAll: false
- label: Job
- multi: false
- name: job
- options: []
- query: 'label_values(kafka_consumergroup_current_offset, job)'
- refresh: 1
- regex: ''
- skipUrlSync: false
- sort: 0
- tagValuesQuery: ''
- tags: []
- tagsQuery: ''
- type: query
- useTags: false
- - allValue: null
- datasource: Prometheus
- definition: ''
- hide: 0
- includeAll: false
- label: Instance
- multi: false
- name: instance
- options: []
- query: >-
- 'label_values(kafka_consumergroup_current_offset{job=~"$job"},'
- 'instance)'
- refresh: 1
- regex: ''
- skipUrlSync: false
- sort: 0
- tagValuesQuery: ''
- tags: []
- tagsQuery: ''
- type: query
- useTags: false
- - allValue: null
- current:
- tags: []
- text: All
- value:
- - $__all
- datasource: Prometheus
- definition: ''
- hide: 0
- includeAll: true
- label: Topic
- multi: true
- name: topic
- options: []
- query: >-
- label_values(kafka_topic_partition_current_offset{instance='$instance',topic!='__consumer_offsets',topic!='--kafka'},
- topic)
- refresh: 1
- regex: ''
- skipUrlSync: false
- sort: 1
- tagValuesQuery: ''
- tags: []
- tagsQuery: topic
- type: query
- useTags: false
-time:
- from: now-1h
- to: now
-timepicker:
- refresh_intervals:
- - 5s
- - 10s
- - 30s
- - 1m
- - 5m
- - 15m
- - 30m
- - 1h
- - 2h
- - 1d
- time_options:
- - 5m
- - 15m
- - 1h
- - 6h
- - 12h
- - 24h
- - 2d
- - 7d
- - 30d
-timezone: browser
-title: Kafka
-uid: jwPKIsniz
-version: 2