Refreshing osm_metrics stack
Moves Prometheus installation to OSM Core for it to be available for MON Collector.
Sets Prometheus version to v2.4.3
Copies prometheus.yml file to the working dir
Modifies configurations in Prometheus to leverage new MON exporter.
Modifies OSM Sample Dashboard at Grafana to point to new metric names.
Removes "Kafka Exporter" container (previous workaround to get metrics from bus)
Signed-off-by: lavado <glavado@whitestack.com>
Change-Id: I15451f717d9ee1cfbdee266a84ed29398360ed94
diff --git a/installers/docker/docker-compose.yaml b/installers/docker/docker-compose.yaml
index c0ca27d..e45fabd 100644
--- a/installers/docker/docker-compose.yaml
+++ b/installers/docker/docker-compose.yaml
@@ -46,6 +46,15 @@
- netOSM
volumes:
- mongo_db:/data/db
+ prometheus:
+ image: prom/prometheus:${PROMETHEUS_TAG:-latest}
+ hostname: prometheus
+ ports:
+ - "${OSM_PROM_PORTS:-9091:9090}"
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+ networks:
+ - netOSM
keystone-db:
image: mariadb:10
networks:
diff --git a/installers/docker/osm_metrics/docker-compose.yml b/installers/docker/osm_metrics/docker-compose.yml
index f529a01..fb0e835 100644
--- a/installers/docker/osm_metrics/docker-compose.yml
+++ b/installers/docker/osm_metrics/docker-compose.yml
@@ -4,22 +4,6 @@
external:
name: ${OSM_NETWORK:-netOSM}
services:
- kafka-exporter:
- image: osm/kafka-exporter
- hostname: kafka-exporter
- ports:
- - "12340:12340"
- networks:
- - netOSM
- prometheus:
- image: prom/prometheus
- hostname: prometheus
- volumes:
- - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- networks:
- - netOSM
- depends_on:
- - kafka-exporter
grafana:
image: grafana/grafana
volumes:
@@ -31,5 +15,3 @@
- "${OSM_PM_PORTS:-3000:3000}"
networks:
- netOSM
- depends_on:
- - prometheus
diff --git a/installers/docker/osm_metrics/kafka-exporter/Dockerfile b/installers/docker/osm_metrics/kafka-exporter/Dockerfile
deleted file mode 100644
index c2a7ef7..0000000
--- a/installers/docker/osm_metrics/kafka-exporter/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM phusion/baseimage
-MAINTAINER Gianpietro Lavado "glavado@whitestack.com"
-EXPOSE 12340
-RUN apt-get update && apt-get install -y python python-pip default-jdk maven git
-RUN pip install pyaml && pip install kafka
-RUN mkdir -p /kafka-topic-exporter/config
-RUN cd /tmp && git clone https://github.com/ogibayashi/kafka-topic-exporter.git && cd /tmp/kafka-topic-exporter/ && git checkout v0.0.5 && mvn install
-RUN cp /tmp/kafka-topic-exporter/target/kafka-topic-exporter-0.0.5-jar-with-dependencies.jar /kafka-topic-exporter/
-RUN rm -rf /tmp/kafka-topic-exporter && apt-get remove -y maven git
-COPY kafka-topic-exporter.properties /kafka-topic-exporter/config/kafka-topic-exporter.properties
-COPY mon_to_kafka_exporter.py /kafka-topic-exporter/mon_to_kafka_exporter.py
-COPY initscript.sh /kafka-topic-exporter/initscript.sh
-WORKDIR /kafka-topic-exporter
-ENTRYPOINT ["./initscript.sh"]
diff --git a/installers/docker/osm_metrics/kafka-exporter/initscript.sh b/installers/docker/osm_metrics/kafka-exporter/initscript.sh
deleted file mode 100755
index 8bbea9e..0000000
--- a/installers/docker/osm_metrics/kafka-exporter/initscript.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# metric_response topic to kafka_exporter_topic
-nohup python /kafka-topic-exporter/mon_to_kafka_exporter.py kafka:9092 &
-
-# kafka_exporter_topic to prometheus web service
-java -jar /kafka-topic-exporter/kafka-topic-exporter-0.0.5-jar-with-dependencies.jar /kafka-topic-exporter/config/kafka-topic-exporter.properties
diff --git a/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties b/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties
deleted file mode 100755
index 755f8af..0000000
--- a/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties
+++ /dev/null
@@ -1,6 +0,0 @@
-exporter.port=12340
-exporter.metric.expire.seconds=10
-kafka.consumer.topics=kafka_exporter_topic
-bootstrap.servers=kafka:9092
-group.id=kte-group
-client.id=kte
diff --git a/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py b/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py
deleted file mode 100644
index e3dbf0e..0000000
--- a/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from kafka import KafkaConsumer, KafkaProducer
-from kafka.errors import KafkaError
-import logging
-import yaml
-import json
-import sys
-import re
-import datetime
-import time
-
-logging.basicConfig(stream=sys.stdout,
- format='%(asctime)s %(message)s',
- datefmt='%m/%d/%Y %I:%M:%S %p',
- level=logging.INFO)
-log = logging.getLogger(__name__)
-
-
-def main():
- if len(sys.argv) <= 1:
- print ("Usage: metric-transformer.py kafka_server")
- exit()
- kafka_server = sys.argv.pop(1)
- kafka_host = kafka_server.split(':')[0]
- kafka_port = kafka_server.split(':')[1]
- transform_messages(kafka_host=kafka_host,
- kafka_port=kafka_port)
-
-
-def transform_messages(kafka_host, kafka_port):
- bootstrap_servers = '{}:{}'.format(kafka_host, kafka_port)
- producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
- key_serializer=str.encode,
- value_serializer=str.encode)
- consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers,
- key_deserializer=str.encode,
- value_deserializer=str.encode)
- consumer.subscribe(["metric_response"])
- for message in consumer:
- try:
- if message.topic == "metric_response":
- if message.key == "read_metric_data_response":
- values = json.loads(message.value)
- new_msg = {
- 'name': values['metric_name'],
- 'value': values['metrics_data']['metrics_series'][-1],
- 'labels': {
- 'resource_uuid': values['resource_uuid']
- }
- }
- log.info("Message to kafka exporter: %s", new_msg)
- future = producer.send(topic='kafka_exporter_topic', key='kafka-exporter-key',
- value=json.dumps(new_msg))
- response = future.get()
- log.info("Response from Kafka: %s", response)
- except Exception as e:
- log.exception("Error processing message: ")
-
-
-if __name__ == '__main__':
- main()
-
diff --git a/installers/docker/osm_metrics/osm-sample-dashboard.json b/installers/docker/osm_metrics/osm-sample-dashboard.json
index 35a165d..a640c50 100644
--- a/installers/docker/osm_metrics/osm-sample-dashboard.json
+++ b/installers/docker/osm_metrics/osm-sample-dashboard.json
@@ -85,11 +85,11 @@
"steppedLine": false,
"targets": [
{
- "expr": "kafka_exporter_topic_cpu_utilization",
+ "expr": "cpu_utilization",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{resource_uuid}}",
+ "legendFormat": "{{vdu_name}}",
"refId": "A"
}
],
@@ -166,11 +166,11 @@
"steppedLine": false,
"targets": [
{
- "expr": "kafka_exporter_topic_average_memory_utilization",
+ "expr": "average_memory_utilization",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{resource_uuid}}",
+ "legendFormat": "{{vdu_name}}",
"refId": "A"
}
],
diff --git a/installers/docker/osm_metrics/prometheus.yml b/installers/docker/prometheus.yml
similarity index 97%
rename from installers/docker/osm_metrics/prometheus.yml
rename to installers/docker/prometheus.yml
index ff1e176..2d535a0 100644
--- a/installers/docker/osm_metrics/prometheus.yml
+++ b/installers/docker/prometheus.yml
@@ -42,4 +42,4 @@
# scheme defaults to 'http'.
static_configs:
- - targets: ['kafka-exporter:12340']
+ - targets: ['mon:8000']