From 7190648dd1913e4aa893aa39ce4942f57f498d28 Mon Sep 17 00:00:00 2001 From: lavado Date: Mon, 13 Apr 2020 17:02:00 -0500 Subject: [PATCH] feature(prometheus): Configuration can be dynamically replaced Adds '--web.enable-lifecycle' option to reload config files. Includes other minor fixes in metrics stack. Change-Id: I88ffe653a08c7104cffb957f934698bf56e64bd9 Signed-off-by: lavado --- installers/docker/docker-compose.yaml | 5 ++++- .../{files => grafana}/dashboards-osm.yml | 0 .../datasource-prometheus.yml | 0 .../osm-sample-dashboard.json | 0 .../osm-system-dashboard.json | 0 installers/docker/osm_pods/prometheus.yaml | 5 ++++- .../{files => prometheus}/prometheus.yml | 2 +- .../node_exporter.service | 0 installers/full_install_osm.sh | 20 +++++++++++++------ 9 files changed, 23 insertions(+), 9 deletions(-) rename installers/docker/{files => grafana}/dashboards-osm.yml (100%) rename installers/docker/{files => grafana}/datasource-prometheus.yml (100%) rename installers/docker/{files => grafana}/osm-sample-dashboard.json (100%) rename installers/docker/{files => grafana}/osm-system-dashboard.json (100%) rename installers/docker/{files => prometheus}/prometheus.yml (95%) rename installers/docker/{files => prometheus_exporters}/node_exporter.service (100%) diff --git a/installers/docker/docker-compose.yaml b/installers/docker/docker-compose.yaml index 104de48b..3fdfbc1a 100644 --- a/installers/docker/docker-compose.yaml +++ b/installers/docker/docker-compose.yaml @@ -88,8 +88,11 @@ services: ports: - "${OSM_PROM_PORTS:-9091:9090}" volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/:/etc/prometheus/ - prom_db:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.enable-lifecycle' networks: - netOSM logging: diff --git a/installers/docker/files/dashboards-osm.yml b/installers/docker/grafana/dashboards-osm.yml similarity index 100% rename from installers/docker/files/dashboards-osm.yml rename to installers/docker/grafana/dashboards-osm.yml diff --git a/installers/docker/files/datasource-prometheus.yml b/installers/docker/grafana/datasource-prometheus.yml similarity index 100% rename from installers/docker/files/datasource-prometheus.yml rename to installers/docker/grafana/datasource-prometheus.yml diff --git a/installers/docker/files/osm-sample-dashboard.json b/installers/docker/grafana/osm-sample-dashboard.json similarity index 100% rename from installers/docker/files/osm-sample-dashboard.json rename to installers/docker/grafana/osm-sample-dashboard.json diff --git a/installers/docker/files/osm-system-dashboard.json b/installers/docker/grafana/osm-system-dashboard.json similarity index 100% rename from installers/docker/files/osm-system-dashboard.json rename to installers/docker/grafana/osm-system-dashboard.json diff --git a/installers/docker/osm_pods/prometheus.yaml b/installers/docker/osm_pods/prometheus.yaml index cdfd94d8..b6ab986c 100644 --- a/installers/docker/osm_pods/prometheus.yaml +++ b/installers/docker/osm_pods/prometheus.yaml @@ -53,7 +53,7 @@ data: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' + - job_name: 'mon_exporter' # metrics_path defaults to '/metrics' # scheme defaults to 'http'. @@ -84,6 +84,9 @@ spec: containers: - name: prometheus image: prom/prometheus:v2.4.3 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-lifecycle ports: - containerPort: 9090 protocol: TCP diff --git a/installers/docker/files/prometheus.yml b/installers/docker/prometheus/prometheus.yml similarity index 95% rename from installers/docker/files/prometheus.yml rename to installers/docker/prometheus/prometheus.yml index a36a241b..8d378379 100644 --- a/installers/docker/files/prometheus.yml +++ b/installers/docker/prometheus/prometheus.yml @@ -36,7 +36,7 @@ rule_files: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' + - job_name: 'mon_exporter' static_configs: - targets: ['mon:8000'] - job_name: 'node_exporter' diff --git a/installers/docker/files/node_exporter.service b/installers/docker/prometheus_exporters/node_exporter.service similarity index 100% rename from installers/docker/files/node_exporter.service rename to installers/docker/prometheus_exporters/node_exporter.service diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 9e4132ca..2994398e 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -312,7 +312,7 @@ function install_prometheus_nodeexporter(){ sudo cp /tmp/node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64/node_exporter /usr/local/bin sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter sudo rm -rf node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64* - sudo cp ${OSM_DEVOPS}/installers/docker/files/node_exporter.service /etc/systemd/system/node_exporter.service + sudo cp ${OSM_DEVOPS}/installers/docker/prometheus/node_exporter.service /etc/systemd/system/node_exporter.service sudo systemctl daemon-reload sudo systemctl restart node_exporter sudo systemctl enable node_exporter @@ -521,12 +521,20 @@ function generate_docker_env_files() { # Docker-compose $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml - # Prometheus - $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus.yml + # Prometheus files + $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/prometheus + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus/prometheus.yml - # Grafana & Prometheus Exporter files - $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/files - $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/* $OSM_DOCKER_WORK_DIR/files/ + # Grafana files + $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/grafana + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/dashboards-osm.yml $OSM_DOCKER_WORK_DIR/grafana/dashboards-osm.yml + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/datasource-prometheus.yml $OSM_DOCKER_WORK_DIR/grafana/datasource-prometheus.yml + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-sample-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-sample-dashboard.json + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-system-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-system-dashboard.json + + # Prometheus Exporters files + $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/prometheus_exporters + $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus_exporters/node_exporter.service $OSM_DOCKER_WORK_DIR/prometheus_exporters/node_exporter.service fi # LCM -- 2.25.1