From de440ed8935a4ad8b7acaad9a6356cedada7bf2b Mon Sep 17 00:00:00 2001 From: David Garcia Date: Mon, 11 Oct 2021 19:56:53 +0200 Subject: [PATCH] (charmed-osm) Add auth to prometheus and update kafka/zk - Authentication to prometheus: - Update prometheus relation to provide auth credentials (user/pass) - Update grafana datasource to include credentials to prometheus - Update kafka and zookeeper charms: - Add support for the charms to talk to the new kafka charm - Add kafka and zookeeper charms in local bundle Change-Id: Ifb83769583178f3a2069ae6756aa430ec69eecce Signed-off-by: David Garcia --- installers/charm/grafana/.yamllint.yaml | 1 + installers/charm/grafana/src/charm.py | 8 +++++ .../templates/default_datasources.yaml | 3 ++ installers/charm/kafka-exporter/src/charm.py | 5 ++- installers/charm/lcm/src/charm.py | 5 ++- installers/charm/local_osm_bundle.yaml | 34 +++++++++++-------- installers/charm/mon/src/charm.py | 14 +++++++- installers/charm/nbi/src/charm.py | 5 ++- installers/charm/pla/src/charm.py | 5 ++- installers/charm/pol/src/charm.py | 5 ++- installers/charm/prometheus/src/charm.py | 8 ++++- .../charm/prometheus/tests/test_charm.py | 4 +++ installers/charm/release_edge.sh | 25 ++++++++++++-- installers/charm/ro/src/charm.py | 5 ++- 14 files changed, 102 insertions(+), 25 deletions(-) diff --git a/installers/charm/grafana/.yamllint.yaml b/installers/charm/grafana/.yamllint.yaml index d71fb69f..783a81d3 100644 --- a/installers/charm/grafana/.yamllint.yaml +++ b/installers/charm/grafana/.yamllint.yaml @@ -32,3 +32,4 @@ ignore: | build/ venv release/ + templates/ diff --git a/installers/charm/grafana/src/charm.py b/installers/charm/grafana/src/charm.py index 36bf6961..778b5eb5 100755 --- a/installers/charm/grafana/src/charm.py +++ b/installers/charm/grafana/src/charm.py @@ -145,11 +145,19 @@ class GrafanaCharm(CharmedOsmBase): def _build_datasources_files(self): files_builder = FilesV3Builder() + prometheus_user = self.prometheus_client.user + prometheus_password = self.prometheus_client.password + enable_basic_auth = all([prometheus_user, prometheus_password]) files_builder.add_file( "datasource_prometheus.yaml", Template(Path("templates/default_datasources.yaml").read_text()).substitute( prometheus_host=self.prometheus_client.hostname, prometheus_port=self.prometheus_client.port, + enable_basic_auth=enable_basic_auth, + user=f"user: {prometheus_user}" if enable_basic_auth else "", + password=f"password: {prometheus_password}" + if enable_basic_auth + else "", ), ) return files_builder.build() diff --git a/installers/charm/grafana/templates/default_datasources.yaml b/installers/charm/grafana/templates/default_datasources.yaml index 9dc0b8fe..88e97dfb 100644 --- a/installers/charm/grafana/templates/default_datasources.yaml +++ b/installers/charm/grafana/templates/default_datasources.yaml @@ -29,3 +29,6 @@ datasources: type: prometheus version: 1 url: http://$prometheus_host:$prometheus_port + basic_auth: $enable_basic_auth + $user + $password diff --git a/installers/charm/kafka-exporter/src/charm.py b/installers/charm/kafka-exporter/src/charm.py index 97ab3d01..1316a4df 100755 --- a/installers/charm/kafka-exporter/src/charm.py +++ b/installers/charm/kafka-exporter/src/charm.py @@ -152,7 +152,10 @@ class KafkaExporterCharm(CharmedOsmBase): """ missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if missing_relations: diff --git a/installers/charm/lcm/src/charm.py b/installers/charm/lcm/src/charm.py index 2fb90e85..5291c077 100755 --- a/installers/charm/lcm/src/charm.py +++ b/installers/charm/lcm/src/charm.py @@ -163,7 +163,10 @@ class LcmCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") diff --git a/installers/charm/local_osm_bundle.yaml b/installers/charm/local_osm_bundle.yaml index e8198ebd..216718de 100644 --- a/installers/charm/local_osm_bundle.yaml +++ b/installers/charm/local_osm_bundle.yaml @@ -14,13 +14,14 @@ description: Single instance OSM bundle bundle: kubernetes applications: - zookeeper-k8s: - charm: "cs:~charmed-osm/zookeeper-k8s" - channel: "stable" + zookeeper: + charm: "./zookeeper/zookeeper.charm" scale: 1 series: kubernetes storage: database: 100M + resources: + image: rocks.canonical.com:443/k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10 annotations: gui-x: 0 gui-y: 550 @@ -38,13 +39,14 @@ applications: annotations: gui-x: -250 gui-y: -200 - kafka-k8s: - charm: "cs:~charmed-osm/kafka-k8s" - channel: "stable" + kafka: + charm: "./kafka/kafka.charm" scale: 1 series: kubernetes storage: database: 100M + resources: + image: rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1 annotations: gui-x: 0 gui-y: 300 @@ -119,7 +121,7 @@ applications: gui-x: -250 gui-y: 550 pla: - charm: "./pla/build" + charm: "./pla/pla.charm" scale: 1 series: kubernetes resources: @@ -164,36 +166,36 @@ applications: relations: - - grafana:prometheus - prometheus:prometheus - - - kafka-k8s:zookeeper - - zookeeper-k8s:zookeeper + - - kafka:zookeeper + - zookeeper:zookeeper - - keystone:db - mariadb-k8s:mysql - - lcm:kafka - - kafka-k8s:kafka + - kafka:kafka - - lcm:mongodb - mongodb:database - - ro:ro - lcm:ro - - ro:kafka - - kafka-k8s:kafka + - kafka:kafka - - ro:mongodb - mongodb:database - - pol:kafka - - kafka-k8s:kafka + - kafka:kafka - - pol:mongodb - mongodb:database - - mon:mongodb - mongodb:database - - mon:kafka - - kafka-k8s:kafka + - kafka:kafka - - pla:kafka - - kafka-k8s:kafka + - kafka:kafka - - pla:mongodb - mongodb:database - - nbi:mongodb - mongodb:database - - nbi:kafka - - kafka-k8s:kafka + - kafka:kafka - - nbi:prometheus - prometheus:prometheus - - nbi:keystone @@ -206,3 +208,5 @@ relations: - keystone:keystone - - mariadb-k8s:mysql - pol:mysql + - - mariadb-k8s:mysql + - grafana:db diff --git a/installers/charm/mon/src/charm.py b/installers/charm/mon/src/charm.py index 917b54a9..85d1fa4e 100755 --- a/installers/charm/mon/src/charm.py +++ b/installers/charm/mon/src/charm.py @@ -157,7 +157,10 @@ class MonCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -260,6 +263,15 @@ class MonCharm(CharmedOsmBase): "OSMMON_PROMETHEUS_URL": f"http://{self.prometheus_client.hostname}:{self.prometheus_client.port}", } ) + prometheus_user = self.prometheus_client.user + prometheus_password = self.prometheus_client.password + if prometheus_user and prometheus_password: + container_builder.add_envs( + { + "OSMMON_PROMETHEUS_USER": prometheus_user, + "OSMMON_PROMETHEUS_PASSWORD": prometheus_password, + } + ) container_builder.add_secret_envs( secret_name=mongodb_secret_name, envs={ diff --git a/installers/charm/nbi/src/charm.py b/installers/charm/nbi/src/charm.py index f9088ab7..c6a7c1ea 100755 --- a/installers/charm/nbi/src/charm.py +++ b/installers/charm/nbi/src/charm.py @@ -162,7 +162,10 @@ class NbiCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") diff --git a/installers/charm/pla/src/charm.py b/installers/charm/pla/src/charm.py index 7867991b..3238dde5 100755 --- a/installers/charm/pla/src/charm.py +++ b/installers/charm/pla/src/charm.py @@ -90,7 +90,10 @@ class PlaCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") diff --git a/installers/charm/pol/src/charm.py b/installers/charm/pol/src/charm.py index 345a87f4..1ac1aa84 100755 --- a/installers/charm/pol/src/charm.py +++ b/installers/charm/pol/src/charm.py @@ -111,7 +111,10 @@ class PolCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") diff --git a/installers/charm/prometheus/src/charm.py b/installers/charm/prometheus/src/charm.py index 40c4f4eb..af39a13a 100755 --- a/installers/charm/prometheus/src/charm.py +++ b/installers/charm/prometheus/src/charm.py @@ -128,7 +128,13 @@ class PrometheusCharm(CharmedOsmBase): ) def _publish_prometheus_info(self, event: EventBase) -> NoReturn: - self.prometheus.publish_info(self.app.name, PORT) + config = ConfigModel(**dict(self.config)) + self.prometheus.publish_info( + self.app.name, + PORT, + user=config.web_config_username, + password=config.web_config_password, + ) def _on_backup_action(self, event: EventBase) -> NoReturn: url = f"http://{self.model.app.name}:{PORT}/api/v1/admin/tsdb/snapshot" diff --git a/installers/charm/prometheus/tests/test_charm.py b/installers/charm/prometheus/tests/test_charm.py index 66e199d2..687c38a6 100644 --- a/installers/charm/prometheus/tests/test_charm.py +++ b/installers/charm/prometheus/tests/test_charm.py @@ -47,6 +47,8 @@ class TestCharm(unittest.TestCase): "site_url": "https://prometheus.192.168.100.100.nip.io", "cluster_issuer": "vault-issuer", "enable_web_admin_api": False, + "web_config_username": "admin", + "web_config_password": "1234", } self.harness.update_config(self.config) @@ -77,6 +79,8 @@ class TestCharm(unittest.TestCase): expected_result = { "hostname": "prometheus", "port": "9090", + "user": "admin", + "password": "1234", } relation_id = self.harness.add_relation("prometheus", "mon") diff --git a/installers/charm/release_edge.sh b/installers/charm/release_edge.sh index c59b2d58..67d0b316 100755 --- a/installers/charm/release_edge.sh +++ b/installers/charm/release_edge.sh @@ -65,9 +65,30 @@ resources_string="--resource image-$image_revision_num" charm release --channel $channel $cs_revision $resources_string echo "$charm charm released!" + +charm="zookeeper" +echo "Releasing $charm charm" +cs_revision=$(charm push $charm/$charm.charm cs:~charmed-osm/$charm | tail -n +1 | head -1 | awk '{print $2}') +resource_revision=$(charm attach $cs_revision image=external::rocks.canonical.com:443/k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10 | tail -n +1 | sed 's/[^0-9]*//g') +image_revision_num=$(echo $resource_revision | awk '{print $NF}') +resources_string="--resource image-$image_revision_num" +charm release --channel $channel $cs_revision $resources_string +echo "$charm charm released!" + + +charm="kafka" +echo "Releasing $charm charm" +cs_revision=$(charm push $charm/$charm.charm cs:~charmed-osm/$charm | tail -n +1 | head -1 | awk '{print $2}') +resource_revision=$(charm attach $cs_revision image=external::rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1 | tail -n +1 | sed 's/[^0-9]*//g') +image_revision_num=$(echo $resource_revision | awk '{print $NF}') +resources_string="--resource image-$image_revision_num" +charm release --channel $channel $cs_revision $resources_string +echo "$charm charm released!" + + # 3. Grant permissions -all_charms="ng-ui nbi pla keystone ro lcm mon pol grafana prometheus mongodb-exporter kafka-exporter mysqld-exporter" +all_charms="ng-ui nbi pla keystone ro lcm mon pol grafana prometheus mongodb-exporter kafka-exporter mysqld-exporter zookeeper kafka" for charm in $all_charms; do echo "Granting permission for $charm charm" charm grant cs:~charmed-osm/$charm --channel $channel --acl read everyone -done \ No newline at end of file +done diff --git a/installers/charm/ro/src/charm.py b/installers/charm/ro/src/charm.py index 2a8c110d..67ba5181 100755 --- a/installers/charm/ro/src/charm.py +++ b/installers/charm/ro/src/charm.py @@ -168,7 +168,10 @@ class RoCharm(CharmedOsmBase): missing_relations = [] if config.enable_ng_ro: - if self.kafka_client.is_missing_data_in_unit(): + if ( + self.kafka_client.is_missing_data_in_unit() + and self.kafka_client.is_missing_data_in_app() + ): missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") -- 2.25.1