From 2cc0ea8e105a42d45af5064faff0b7d27fb40fc0 Mon Sep 17 00:00:00 2001 From: lavado Date: Tue, 15 May 2018 03:24:03 +0000 Subject: [PATCH] Adding composers and customization files for ELK sample system and Metrics sample system (Prometheus & Grafana) Change-Id: I2ba4ce27119051f210aa13a5a49f859c0b157d99 Signed-off-by: lavado --- installers/docker/osm_elk/docker-compose.yml | 40 +++ installers/docker/osm_elk/elasticsearch.yml | 5 + installers/docker/osm_elk/kibana.yml | 4 + installers/docker/osm_elk/logstash.conf | 17 ++ installers/docker/osm_elk/logstash.yml | 3 + .../docker/osm_metrics/dashboards-osm.yml | 10 + .../osm_metrics/datasource-prometheus.yml | 9 + .../docker/osm_metrics/docker-compose.yml | 35 +++ .../osm_metrics/kafka-exporter/Dockerfile | 14 + .../osm_metrics/kafka-exporter/initscript.sh | 7 + .../kafka-topic-exporter.properties | 6 + .../kafka-exporter/mon_to_kafka_exporter.py | 61 +++++ .../osm_metrics/osm-sample-dashboard.json | 253 ++++++++++++++++++ installers/docker/osm_metrics/prometheus.yml | 45 ++++ 14 files changed, 509 insertions(+) create mode 100644 installers/docker/osm_elk/docker-compose.yml create mode 100644 installers/docker/osm_elk/elasticsearch.yml create mode 100644 installers/docker/osm_elk/kibana.yml create mode 100644 installers/docker/osm_elk/logstash.conf create mode 100644 installers/docker/osm_elk/logstash.yml create mode 100644 installers/docker/osm_metrics/dashboards-osm.yml create mode 100644 installers/docker/osm_metrics/datasource-prometheus.yml create mode 100644 installers/docker/osm_metrics/docker-compose.yml create mode 100644 installers/docker/osm_metrics/kafka-exporter/Dockerfile create mode 100755 installers/docker/osm_metrics/kafka-exporter/initscript.sh create mode 100755 installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties create mode 100644 installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py create mode 100644 installers/docker/osm_metrics/osm-sample-dashboard.json create mode 100644 installers/docker/osm_metrics/prometheus.yml diff --git a/installers/docker/osm_elk/docker-compose.yml b/installers/docker/osm_elk/docker-compose.yml new file mode 100644 index 00000000..222a1d03 --- /dev/null +++ b/installers/docker/osm_elk/docker-compose.yml @@ -0,0 +1,40 @@ +version: '3' + +services: + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.3 + volumes: + - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + + logstash: + image: docker.elastic.co/logstash/logstash-oss:6.2.3 + volumes: + - ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + image: docker.elastic.co/kibana/kibana-oss:6.2.3 + volumes: + - ./kibana.yml:/usr/share/kibana/config/kibana.yml:ro + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + external: + name: netOSM diff --git a/installers/docker/osm_elk/elasticsearch.yml b/installers/docker/osm_elk/elasticsearch.yml new file mode 100644 index 00000000..ba678747 --- /dev/null +++ b/installers/docker/osm_elk/elasticsearch.yml @@ -0,0 +1,5 @@ +--- +network.host: 0.0.0.0 +cluster.name: "docker-cluster" +discovery.type: single-node +discovery.zen.minimum_master_nodes: 1 diff --git a/installers/docker/osm_elk/kibana.yml b/installers/docker/osm_elk/kibana.yml new file mode 100644 index 00000000..a8a39a96 --- /dev/null +++ b/installers/docker/osm_elk/kibana.yml @@ -0,0 +1,4 @@ +--- +server.host: "0" +server.name: kibana +elasticsearch.url: http://elasticsearch:9200 diff --git a/installers/docker/osm_elk/logstash.conf b/installers/docker/osm_elk/logstash.conf new file mode 100644 index 00000000..a83a572b --- /dev/null +++ b/installers/docker/osm_elk/logstash.conf @@ -0,0 +1,17 @@ +input { + tcp { + port => 5000 + } + kafka { + bootstrap_servers => 'kafka:9092' + topics => ["alarm_response", "lcm_pm"] + } +} + +## Add your filters / logstash plugins configuration here + +output { + elasticsearch { + hosts => "elasticsearch:9200" + } +} diff --git a/installers/docker/osm_elk/logstash.yml b/installers/docker/osm_elk/logstash.yml new file mode 100644 index 00000000..91bcbb1c --- /dev/null +++ b/installers/docker/osm_elk/logstash.yml @@ -0,0 +1,3 @@ +--- +path.config: /usr/share/logstash/pipeline +http.host: "0.0.0.0" diff --git a/installers/docker/osm_metrics/dashboards-osm.yml b/installers/docker/osm_metrics/dashboards-osm.yml new file mode 100644 index 00000000..efe7fa1b --- /dev/null +++ b/installers/docker/osm_metrics/dashboards-osm.yml @@ -0,0 +1,10 @@ +# # config file version +apiVersion: 1 + +providers: + - name: 'osm' + orgId: 1 + folder: '' + type: file + options: + path: /etc/grafana/provisioning/dashboards/ diff --git a/installers/docker/osm_metrics/datasource-prometheus.yml b/installers/docker/osm_metrics/datasource-prometheus.yml new file mode 100644 index 00000000..b6151ee4 --- /dev/null +++ b/installers/docker/osm_metrics/datasource-prometheus.yml @@ -0,0 +1,9 @@ +datasources: +- access: 'proxy' # make grafana perform the requests + editable: true # whether it should be editable + is_default: true # whether this should be the default DS + name: 'osm_prometheus' # name of the datasource + org_id: 1 # id of the organization to tie this datasource to + type: 'prometheus' # type of the data source + url: 'http://prometheus:9090' # url of the prom instance + version: 1 # well, versioning diff --git a/installers/docker/osm_metrics/docker-compose.yml b/installers/docker/osm_metrics/docker-compose.yml new file mode 100644 index 00000000..c6832a2b --- /dev/null +++ b/installers/docker/osm_metrics/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3' +networks: + netOSM: + external: true +services: + kafka-exporter: + image: osm/kafka-exporter + hostname: kafka-exporter + ports: + - "12340:12340" + networks: + - netOSM + prometheus: + image: prom/prometheus + hostname: prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + networks: + - netOSM + depends_on: + - kafka-exporter + grafana: + image: grafana/grafana + volumes: + - ./dashboards-osm.yml:/etc/grafana/provisioning/dashboards/dashboards-osm.yml + - ./osm-sample-dashboard.json:/etc/grafana/provisioning/dashboards/osm-sample-dashboard.json + - ./datasource-prometheus.yml:/etc/grafana/provisioning/datasources/datasource-prometheus.yml + hostname: grafana + ports: + - "3000:3000" + networks: + - netOSM + depends_on: + - prometheus + diff --git a/installers/docker/osm_metrics/kafka-exporter/Dockerfile b/installers/docker/osm_metrics/kafka-exporter/Dockerfile new file mode 100644 index 00000000..c2a7ef73 --- /dev/null +++ b/installers/docker/osm_metrics/kafka-exporter/Dockerfile @@ -0,0 +1,14 @@ +FROM phusion/baseimage +MAINTAINER Gianpietro Lavado "glavado@whitestack.com" +EXPOSE 12340 +RUN apt-get update && apt-get install -y python python-pip default-jdk maven git +RUN pip install pyaml && pip install kafka +RUN mkdir -p /kafka-topic-exporter/config +RUN cd /tmp && git clone https://github.com/ogibayashi/kafka-topic-exporter.git && cd /tmp/kafka-topic-exporter/ && git checkout v0.0.5 && mvn install +RUN cp /tmp/kafka-topic-exporter/target/kafka-topic-exporter-0.0.5-jar-with-dependencies.jar /kafka-topic-exporter/ +RUN rm -rf /tmp/kafka-topic-exporter && apt-get remove -y maven git +COPY kafka-topic-exporter.properties /kafka-topic-exporter/config/kafka-topic-exporter.properties +COPY mon_to_kafka_exporter.py /kafka-topic-exporter/mon_to_kafka_exporter.py +COPY initscript.sh /kafka-topic-exporter/initscript.sh +WORKDIR /kafka-topic-exporter +ENTRYPOINT ["./initscript.sh"] diff --git a/installers/docker/osm_metrics/kafka-exporter/initscript.sh b/installers/docker/osm_metrics/kafka-exporter/initscript.sh new file mode 100755 index 00000000..8bbea9e8 --- /dev/null +++ b/installers/docker/osm_metrics/kafka-exporter/initscript.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# metric_response topic to kafka_exporter_topic +nohup python /kafka-topic-exporter/mon_to_kafka_exporter.py kafka:9092 & + +# kafka_exporter_topic to prometheus web service +java -jar /kafka-topic-exporter/kafka-topic-exporter-0.0.5-jar-with-dependencies.jar /kafka-topic-exporter/config/kafka-topic-exporter.properties diff --git a/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties b/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties new file mode 100755 index 00000000..755f8af4 --- /dev/null +++ b/installers/docker/osm_metrics/kafka-exporter/kafka-topic-exporter.properties @@ -0,0 +1,6 @@ +exporter.port=12340 +exporter.metric.expire.seconds=10 +kafka.consumer.topics=kafka_exporter_topic +bootstrap.servers=kafka:9092 +group.id=kte-group +client.id=kte diff --git a/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py b/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py new file mode 100644 index 00000000..e3dbf0ed --- /dev/null +++ b/installers/docker/osm_metrics/kafka-exporter/mon_to_kafka_exporter.py @@ -0,0 +1,61 @@ +from kafka import KafkaConsumer, KafkaProducer +from kafka.errors import KafkaError +import logging +import yaml +import json +import sys +import re +import datetime +import time + +logging.basicConfig(stream=sys.stdout, + format='%(asctime)s %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', + level=logging.INFO) +log = logging.getLogger(__name__) + + +def main(): + if len(sys.argv) <= 1: + print ("Usage: metric-transformer.py kafka_server") + exit() + kafka_server = sys.argv.pop(1) + kafka_host = kafka_server.split(':')[0] + kafka_port = kafka_server.split(':')[1] + transform_messages(kafka_host=kafka_host, + kafka_port=kafka_port) + + +def transform_messages(kafka_host, kafka_port): + bootstrap_servers = '{}:{}'.format(kafka_host, kafka_port) + producer = KafkaProducer(bootstrap_servers=bootstrap_servers, + key_serializer=str.encode, + value_serializer=str.encode) + consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers, + key_deserializer=str.encode, + value_deserializer=str.encode) + consumer.subscribe(["metric_response"]) + for message in consumer: + try: + if message.topic == "metric_response": + if message.key == "read_metric_data_response": + values = json.loads(message.value) + new_msg = { + 'name': values['metric_name'], + 'value': values['metrics_data']['metrics_series'][-1], + 'labels': { + 'resource_uuid': values['resource_uuid'] + } + } + log.info("Message to kafka exporter: %s", new_msg) + future = producer.send(topic='kafka_exporter_topic', key='kafka-exporter-key', + value=json.dumps(new_msg)) + response = future.get() + log.info("Response from Kafka: %s", response) + except Exception as e: + log.exception("Error processing message: ") + + +if __name__ == '__main__': + main() + diff --git a/installers/docker/osm_metrics/osm-sample-dashboard.json b/installers/docker/osm_metrics/osm-sample-dashboard.json new file mode 100644 index 00000000..35a165d5 --- /dev/null +++ b/installers/docker/osm_metrics/osm-sample-dashboard.json @@ -0,0 +1,253 @@ +{ + "__inputs": [ + { + "name": "osm_prometheus", + "label": "osm_prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_exporter_topic_cpu_utilization", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{resource_uuid}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "VDU CPU Metrics", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osm_prometheus", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_exporter_topic_average_memory_utilization", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{resource_uuid}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "VDU Memory Metrics", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OSM Sample Dashboard", + "uid": "x5vtPXmik", + "version": 1 +} diff --git a/installers/docker/osm_metrics/prometheus.yml b/installers/docker/osm_metrics/prometheus.yml new file mode 100644 index 00000000..ff1e1767 --- /dev/null +++ b/installers/docker/osm_metrics/prometheus.yml @@ -0,0 +1,45 @@ +# Copyright 2018 The Prometheus Authors +# Copyright 2018 Whitestack +# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['kafka-exporter:12340'] -- 2.25.1