(charmed-osm) Add auth to prometheus and update kafka/zk

- Authentication to prometheus:
  - Update prometheus relation to provide auth credentials (user/pass)
  - Update grafana datasource to include credentials to prometheus
- Update kafka and zookeeper charms:
  - Add support for the charms to talk to the new kafka charm
  - Add kafka and zookeeper charms in local bundle

Change-Id: Ifb83769583178f3a2069ae6756aa430ec69eecce
Signed-off-by: David Garcia <david.garcia@canonical.com>

installers/charm/grafana/.yamllint.yaml

@@ -32,3 +32,4 @@ ignore: |
   build/
   venv
   release/
+  templates/

installers/charm/grafana/src/charm.py

@@ -145,11 +145,19 @@ class GrafanaCharm(CharmedOsmBase):
 
     def _build_datasources_files(self):
         files_builder = FilesV3Builder()
+        prometheus_user = self.prometheus_client.user
+        prometheus_password = self.prometheus_client.password
+        enable_basic_auth = all([prometheus_user, prometheus_password])
         files_builder.add_file(
             "datasource_prometheus.yaml",
             Template(Path("templates/default_datasources.yaml").read_text()).substitute(
                 prometheus_host=self.prometheus_client.hostname,
                 prometheus_port=self.prometheus_client.port,
+                enable_basic_auth=enable_basic_auth,
+                user=f"user: {prometheus_user}" if enable_basic_auth else "",
+                password=f"password: {prometheus_password}"
+                if enable_basic_auth
+                else "",
             ),
         )
         return files_builder.build()

installers/charm/grafana/templates/default_datasources.yaml

@@ -29,3 +29,6 @@ datasources:
     type: prometheus
     version: 1
     url: http://$prometheus_host:$prometheus_port
+    basic_auth: $enable_basic_auth
+    $user
+    $password

installers/charm/kafka-exporter/src/charm.py

@@ -152,7 +152,10 @@ class KafkaExporterCharm(CharmedOsmBase):
         """
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
 
         if missing_relations:

installers/charm/lcm/src/charm.py

@@ -163,7 +163,10 @@ class LcmCharm(CharmedOsmBase):
     def _check_missing_dependencies(self, config: ConfigModel):
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
         if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
             missing_relations.append("mongodb")

installers/charm/local_osm_bundle.yaml

@@ -14,13 +14,14 @@
 description: Single instance OSM bundle
 bundle: kubernetes
 applications:
-  zookeeper-k8s:
-    charm: "cs:~charmed-osm/zookeeper-k8s"
-    channel: "stable"
+  zookeeper:
+    charm: "./zookeeper/zookeeper.charm"
     scale: 1
     series: kubernetes
     storage:
       database: 100M
+    resources:
+      image: rocks.canonical.com:443/k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10
     annotations:
       gui-x: 0
       gui-y: 550
@@ -38,13 +39,14 @@ applications:
     annotations:
       gui-x: -250
       gui-y: -200
-  kafka-k8s:
-    charm: "cs:~charmed-osm/kafka-k8s"
-    channel: "stable"
+  kafka:
+    charm: "./kafka/kafka.charm"
     scale: 1
     series: kubernetes
     storage:
       database: 100M
+    resources:
+      image: rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1
     annotations:
       gui-x: 0
       gui-y: 300
@@ -119,7 +121,7 @@ applications:
       gui-x: -250
       gui-y: 550
   pla:
-    charm: "./pla/build"
+    charm: "./pla/pla.charm"
     scale: 1
     series: kubernetes
     resources:
@@ -164,36 +166,36 @@ applications:
 relations:
   - - grafana:prometheus
     - prometheus:prometheus
-  - - kafka-k8s:zookeeper
-    - zookeeper-k8s:zookeeper
+  - - kafka:zookeeper
+    - zookeeper:zookeeper
   - - keystone:db
     - mariadb-k8s:mysql
   - - lcm:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - lcm:mongodb
     - mongodb:database
   - - ro:ro
     - lcm:ro
   - - ro:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - ro:mongodb
     - mongodb:database
   - - pol:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - pol:mongodb
     - mongodb:database
   - - mon:mongodb
     - mongodb:database
   - - mon:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - pla:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - pla:mongodb
     - mongodb:database
   - - nbi:mongodb
     - mongodb:database
   - - nbi:kafka
-    - kafka-k8s:kafka
+    - kafka:kafka
   - - nbi:prometheus
     - prometheus:prometheus
   - - nbi:keystone
@@ -206,3 +208,5 @@ relations:
     - keystone:keystone
   - - mariadb-k8s:mysql
     - pol:mysql
+  - - mariadb-k8s:mysql
+    - grafana:db

installers/charm/mon/src/charm.py

@@ -157,7 +157,10 @@ class MonCharm(CharmedOsmBase):
     def _check_missing_dependencies(self, config: ConfigModel):
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
         if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
             missing_relations.append("mongodb")
@@ -260,6 +263,15 @@ class MonCharm(CharmedOsmBase):
                 "OSMMON_PROMETHEUS_URL": f"http://{self.prometheus_client.hostname}:{self.prometheus_client.port}",
             }
         )
+        prometheus_user = self.prometheus_client.user
+        prometheus_password = self.prometheus_client.password
+        if prometheus_user and prometheus_password:
+            container_builder.add_envs(
+                {
+                    "OSMMON_PROMETHEUS_USER": prometheus_user,
+                    "OSMMON_PROMETHEUS_PASSWORD": prometheus_password,
+                }
+            )
         container_builder.add_secret_envs(
             secret_name=mongodb_secret_name,
             envs={

installers/charm/nbi/src/charm.py

@@ -162,7 +162,10 @@ class NbiCharm(CharmedOsmBase):
     def _check_missing_dependencies(self, config: ConfigModel):
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
         if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
             missing_relations.append("mongodb")

installers/charm/pla/src/charm.py

@@ -90,7 +90,10 @@ class PlaCharm(CharmedOsmBase):
     def _check_missing_dependencies(self, config: ConfigModel):
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
         if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
             missing_relations.append("mongodb")

installers/charm/pol/src/charm.py

@@ -111,7 +111,10 @@ class PolCharm(CharmedOsmBase):
     def _check_missing_dependencies(self, config: ConfigModel):
         missing_relations = []
 
-        if self.kafka_client.is_missing_data_in_unit():
+        if (
+            self.kafka_client.is_missing_data_in_unit()
+            and self.kafka_client.is_missing_data_in_app()
+        ):
             missing_relations.append("kafka")
         if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
             missing_relations.append("mongodb")

installers/charm/prometheus/src/charm.py

@@ -128,7 +128,13 @@ class PrometheusCharm(CharmedOsmBase):
         )
 
     def _publish_prometheus_info(self, event: EventBase) -> NoReturn:
-        self.prometheus.publish_info(self.app.name, PORT)
+        config = ConfigModel(**dict(self.config))
+        self.prometheus.publish_info(
+            self.app.name,
+            PORT,
+            user=config.web_config_username,
+            password=config.web_config_password,
+        )
 
     def _on_backup_action(self, event: EventBase) -> NoReturn:
         url = f"http://{self.model.app.name}:{PORT}/api/v1/admin/tsdb/snapshot"

installers/charm/prometheus/tests/test_charm.py

@@ -47,6 +47,8 @@ class TestCharm(unittest.TestCase):
             "site_url": "https://prometheus.192.168.100.100.nip.io",
             "cluster_issuer": "vault-issuer",
             "enable_web_admin_api": False,
+            "web_config_username": "admin",
+            "web_config_password": "1234",
         }
         self.harness.update_config(self.config)
 
@@ -77,6 +79,8 @@ class TestCharm(unittest.TestCase):
         expected_result = {
             "hostname": "prometheus",
             "port": "9090",
+            "user": "admin",
+            "password": "1234",
         }
 
         relation_id = self.harness.add_relation("prometheus", "mon")

installers/charm/release_edge.sh

@@ -65,9 +65,30 @@ resources_string="--resource image-$image_revision_num"
 charm release --channel $channel $cs_revision $resources_string
 echo "$charm charm released!"
 
+
+charm="zookeeper"
+echo "Releasing $charm charm"
+cs_revision=$(charm push $charm/$charm.charm cs:~charmed-osm/$charm | tail -n +1 | head -1 | awk '{print $2}')
+resource_revision=$(charm attach $cs_revision image=external::rocks.canonical.com:443/k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10 | tail -n +1 | sed 's/[^0-9]*//g')
+image_revision_num=$(echo $resource_revision  | awk '{print $NF}')
+resources_string="--resource image-$image_revision_num"
+charm release --channel $channel $cs_revision $resources_string
+echo "$charm charm released!"
+
+
+charm="kafka"
+echo "Releasing $charm charm"
+cs_revision=$(charm push $charm/$charm.charm cs:~charmed-osm/$charm | tail -n +1 | head -1 | awk '{print $2}')
+resource_revision=$(charm attach $cs_revision image=external::rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1 | tail -n +1 | sed 's/[^0-9]*//g')
+image_revision_num=$(echo $resource_revision  | awk '{print $NF}')
+resources_string="--resource image-$image_revision_num"
+charm release --channel $channel $cs_revision $resources_string
+echo "$charm charm released!"
+
+
 # 3. Grant permissions
-all_charms="ng-ui nbi pla keystone ro lcm mon pol grafana prometheus mongodb-exporter kafka-exporter mysqld-exporter"
+all_charms="ng-ui nbi pla keystone ro lcm mon pol grafana prometheus mongodb-exporter kafka-exporter mysqld-exporter zookeeper kafka"
 for charm in $all_charms; do
     echo "Granting permission for $charm charm"
     charm grant cs:~charmed-osm/$charm --channel $channel --acl read everyone
-done
\ No newline at end of file
+done

installers/charm/ro/src/charm.py

@@ -168,7 +168,10 @@ class RoCharm(CharmedOsmBase):
         missing_relations = []
 
         if config.enable_ng_ro:
-            if self.kafka_client.is_missing_data_in_unit():
+            if (
+                self.kafka_client.is_missing_data_in_unit()
+                and self.kafka_client.is_missing_data_in_app()
+            ):
                 missing_relations.append("kafka")
             if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit():
                 missing_relations.append("mongodb")