Adding Kafka charm 87/10187/1
authorsousaedu <eduardo.sousa@canonical.com>
Mon, 18 Jan 2021 12:27:42 +0000 (12:27 +0000)
committersousaedu <eduardo.sousa@canonical.com>
Mon, 18 Jan 2021 12:27:42 +0000 (12:27 +0000)
Change-Id: Ie1e55c79d5cab9bd51bed91f378abce37f31218b
Signed-off-by: sousaedu <eduardo.sousa@canonical.com>
15 files changed:
installers/charm/kafka-k8s/.gitignore [new file with mode: 0644]
installers/charm/kafka-k8s/.yamllint.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/README.md [new file with mode: 0755]
installers/charm/kafka-k8s/config.yaml [new file with mode: 0755]
installers/charm/kafka-k8s/icon.svg [new file with mode: 0644]
installers/charm/kafka-k8s/layer.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/metadata.yaml [new file with mode: 0755]
installers/charm/kafka-k8s/reactive/kafka.py [new file with mode: 0644]
installers/charm/kafka-k8s/reactive/spec_template.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/test-requirements.txt [new file with mode: 0644]
installers/charm/kafka-k8s/tests/basic_deployment.py [new file with mode: 0644]
installers/charm/kafka-k8s/tests/bundles/kafka-ha.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/tests/bundles/kafka.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/tests/tests.yaml [new file with mode: 0644]
installers/charm/kafka-k8s/tox.ini [new file with mode: 0644]

diff --git a/installers/charm/kafka-k8s/.gitignore b/installers/charm/kafka-k8s/.gitignore
new file mode 100644 (file)
index 0000000..712eb96
--- /dev/null
@@ -0,0 +1,24 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+release/
+__pycache__
+.tox
diff --git a/installers/charm/kafka-k8s/.yamllint.yaml b/installers/charm/kafka-k8s/.yamllint.yaml
new file mode 100644 (file)
index 0000000..21b95b5
--- /dev/null
@@ -0,0 +1,34 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+---
+
+extends: default
+rules:
+  line-length: disable
+yaml-files:
+  - '*.yaml'
+  - '*.yml'
+  - '.yamllint'
+ignore: |
+ reactive/
+ .tox
+ release/
diff --git a/installers/charm/kafka-k8s/README.md b/installers/charm/kafka-k8s/README.md
new file mode 100755 (executable)
index 0000000..b5283a9
--- /dev/null
@@ -0,0 +1,107 @@
+<!-- Copyright 2021 Canonical Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+
+For those usages not covered by the Apache License, Version 2.0 please
+contact: legal@canonical.com
+
+To get in touch with the maintainers, please contact:
+osm-charmers@lists.launchpad.net -->
+
+# Overview
+
+Kafka for Juju CAAS
+
+
+## Testing
+
+The tests of this charm are done using tox and Zaza.
+
+
+
+### Prepare environment
+
+The machine in which the tests are run needs access to a juju k8s controller. The easiest way to approach this is by executing the following commands:
+
+```
+sudo apt install tox -y
+sudo snap install microk8s --classic
+sudo snap install juju
+
+microk8s.status --wait-ready
+microk8s.enable storage dashboard dns
+
+juju bootstrap microk8s k8s-cloud
+```
+
+If /usr/bin/python does not exist, you should probably need to do this:
+
+```
+sudo ln -s /usr/bin/python3 /usr/bin/python
+```
+
+### Build Charm
+
+**Download dependencies:**
+
+```
+mkdir -p ~/charm/layers ~/charm/build
+cd ~/charm/layers
+git clone https://git.launchpad.net/charm-k8s-kafka kafka-k8s
+git clone https://git.launchpad.net/charm-k8s-zookeeper zookeeper-k8s
+git clone https://git.launchpad.net/charm-osm-common osm-common
+```
+
+**Charm structure:**
+
+```
+├── config.yaml
+├── icon.svg
+├── layer.yaml
+├── metadata.yaml
+├── reactive
+│   ├── spec_template.yaml
+│   └── kafka.py
+├── README.md
+├── test-requirements.txt
+├── tests
+│   ├── basic_deployment.py
+│   ├── bundles
+│   │   ├── kafka-ha.yaml
+│   │   └── kafka.yaml
+│   └── tests.yaml
+└── tox.ini
+```
+
+**Setup environment variables:**
+
+```
+export CHARM_LAYERS_DIR=~/charm/layers
+export CHARM_BUILD_DIR=~/charm/build
+```
+
+**Build:**
+
+```
+mkdir ~/charm/layers/kafka-k8s/tests/build
+charm build ~/charm/layers/kafka-k8s
+charm build ~/charm/layers/zookeeper-k8s
+mv ~/charm/build/* ~/charm/layers/kafka-k8s/tests/build/
+```
+
+### Test charm with Tox
+
+```
+cd ~/charm/layers/kafka-k8s
+tox -e func
+```
\ No newline at end of file
diff --git a/installers/charm/kafka-k8s/config.yaml b/installers/charm/kafka-k8s/config.yaml
new file mode 100755 (executable)
index 0000000..3f925ce
--- /dev/null
@@ -0,0 +1,50 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+options:
+    advertised-hostname:
+        description: Kafka Hostname
+        type: string
+        default: "kafka-k8s"
+    advertised-port:
+        description: Kafka port
+        type: int
+        default: 9092
+    kafka-units:
+        description: Kafka kafka-units
+        type: int
+        default: 1
+    zookeeper-units:
+        description: Zookeeper kafka-units
+        type: int
+        default: 1
+    zookeeper-service-name:
+        description: Zookeeper Service name
+        type: string
+        default: zookeeper-k8s-endpoints
+    image:
+        type: string
+        description: OCI image
+        default: rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1
+    num-partitions:
+        description: Kafka number of partitions per topic
+        type: int
+        default: 1
diff --git a/installers/charm/kafka-k8s/icon.svg b/installers/charm/kafka-k8s/icon.svg
new file mode 100644 (file)
index 0000000..de5ab1b
--- /dev/null
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" height="100px" width="100px" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 100 100"><!-- Generator: Sketch 43.2 (39069) - http://www.bohemiancoding.com/sketch -->
+ <title>kafka</title>
+ <desc>Created with Sketch.</desc>
+ <g id="Page-1" fill-rule="evenodd" fill="none">
+  <g id="kafka">
+   <path id="path4784" fill-rule="nonzero" fill="#fff" d="m50 1.1369e-13c-27.614-1.69e-15 -50 22.386-50 50-3.3818e-15 27.614 22.386 50 50 50 13.261 0 25.979-5.268 35.355-14.645 9.377-9.376 14.645-22.094 14.645-35.355 0-27.614-22.386-50-50-50z"/>
+   <g id="kafta" fill="#000" transform="translate(27 13)">
+    <path id="Combined-Shape" d="m13.333 20c-5.5225 0-9.9997-4.477-9.9997-10 0-5.5228 4.4772-10 9.9997-10 5.523 0 10 4.4772 10 10 0 5.523-4.477 10-10 10zm0.278-5c2.608 0 4.722-2.114 4.722-4.722 0-2.6082-2.114-4.7224-4.722-4.7224s-4.7221 2.1142-4.7221 4.7224c0 2.608 2.1141 4.722 4.7221 4.722z"/>
+    <rect id="Rectangle" y="18.889" x="11.111" height="6.6667" width="4.4444"/>
+    <rect id="Rectangle-Copy" y="49.444" x="11.111" height="6.6667" width="4.4444"/>
+    <rect id="Rectangle-Copy-2" transform="translate(26.776 29.702) rotate(60) translate(-26.776 -29.702)" height="6.6667" width="4.4444" y="26.369" x="24.553"/>
+    <rect id="Rectangle-Copy-3" transform="translate(26.22 45.258) scale(1 -1) rotate(60) translate(-26.22 -45.258)" height="6.6667" width="4.4444" y="41.925" x="23.998"/>
+    <path id="Combined-Shape-Copy-4" d="m12.778 50c-7.0572 0-12.778-5.721-12.778-12.778s5.7208-12.778 12.778-12.778c7.057 0 12.778 5.721 12.778 12.778s-5.721 12.778-12.778 12.778zm0.355-6.389c3.332 0 6.034-2.701 6.034-6.034 0-3.332-2.702-6.034-6.034-6.034-3.3327 0-6.0342 2.702-6.0342 6.034 0 3.333 2.7015 6.034 6.0342 6.034z"/>
+    <path id="Combined-Shape-Copy" d="m13.333 75c-5.5225 0-9.9997-4.477-9.9997-10s4.4772-10 9.9997-10c5.523 0 10 4.477 10 10s-4.477 10-10 10zm0.278-5c2.608 0 4.722-2.114 4.722-4.722s-2.114-4.722-4.722-4.722-4.7221 2.114-4.7221 4.722 2.1141 4.722 4.7221 4.722z"/>
+    <path id="Combined-Shape-Copy-2" d="m36.667 33.889c-5.523 0-10-4.477-10-10s4.477-10 10-10 10 4.477 10 10-4.477 10-10 10zm0.277-5c2.608 0 4.723-2.114 4.723-4.722s-2.115-4.723-4.723-4.723-4.722 2.115-4.722 4.723 2.114 4.722 4.722 4.722z"/>
+    <path id="Combined-Shape-Copy-3" d="m36.667 61.111c-5.523 0-10-4.477-10-10s4.477-10 10-10 10 4.477 10 10-4.477 10-10 10zm0.277-5c2.608 0 4.723-2.114 4.723-4.722s-2.115-4.722-4.723-4.722-4.722 2.114-4.722 4.722 2.114 4.722 4.722 4.722z"/>
+   </g>
+  </g>
+ </g>
+</svg>
diff --git a/installers/charm/kafka-k8s/layer.yaml b/installers/charm/kafka-k8s/layer.yaml
new file mode 100644 (file)
index 0000000..d727726
--- /dev/null
@@ -0,0 +1,30 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+includes:
+  - "layer:caas-base"
+  - 'layer:status'
+  - 'layer:leadership'
+  - "layer:osm-common"
+  - 'interface:kafka'
+  - 'interface:zookeeper'
+
+repo: https://code.launchpad.net/osm-k8s-bundle
diff --git a/installers/charm/kafka-k8s/metadata.yaml b/installers/charm/kafka-k8s/metadata.yaml
new file mode 100755 (executable)
index 0000000..7c22ce2
--- /dev/null
@@ -0,0 +1,44 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+name: "kafka-k8s"
+summary: "Kafka charm for Kubernetes."
+maintainers:
+    - "SolutionsQA <solutionsqa@lists.canonical.com>"
+description: |
+  A CAAS charm to deploy kafka.
+tags:
+    - "application"
+series:
+    - "kubernetes"
+provides:
+    kafka:
+        interface: kafka
+requires:
+    zookeeper:
+        interface: zookeeper
+storage:
+    database:
+        type: filesystem
+        location: /var/lib/kafka
+deployment:
+    type: stateful
+    service: cluster
diff --git a/installers/charm/kafka-k8s/reactive/kafka.py b/installers/charm/kafka-k8s/reactive/kafka.py
new file mode 100644 (file)
index 0000000..72ff5bc
--- /dev/null
@@ -0,0 +1,139 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+from charms.layer.caas_base import pod_spec_set
+from charms.reactive import when, when_not, hook
+from charms.reactive import endpoint_from_flag
+from charms.reactive.flags import set_flag, clear_flag
+from charmhelpers.core.hookenv import log, metadata, config
+from charms import layer
+from charms.osm.k8s import get_service_ip
+
+
+@hook("upgrade-charm")
+@when("leadership.is_leader")
+def upgrade():
+    clear_flag("kafka-k8s.configured")
+
+
+@when("config.changed")
+@when("leadership.is_leader")
+def restart():
+    clear_flag("kafka-k8s.configured")
+
+
+@when_not("zookeeper.ready")
+@when("leadership.is_leader")
+def waiting_for_zookeeper():
+    layer.status.waiting("Waiting for Zookeeper to be ready")
+
+
+@when("zookeeper.ready")
+@when_not("kafka-k8s.configured")
+@when("leadership.is_leader")
+def configure():
+    layer.status.maintenance("Configuring kafka container")
+    try:
+        zookeeper = endpoint_from_flag("zookeeper.ready")
+        zk_units = zookeeper.zookeepers()
+        zk_unit = zk_units[0]
+        if zk_unit["port"]:
+            cfg = config()
+            zookeeper_uri = ""
+            zk_pod_base_name = "zookeeper-k8s"
+            zk_service_name = cfg.get("zookeeper-service-name")
+            zk_num = cfg.get("zookeeper-units")
+            if zk_num == 1:
+                zookeeper_uri = "{}:{}".format(zk_unit["host"], zk_unit["port"])
+            else:
+                for i in range(0, zk_num):
+                    if i:
+                        zookeeper_uri += ","
+                    zookeeper_uri += "{}-{}.{}:{}".format(
+                        zk_pod_base_name, i, zk_service_name, zk_unit["port"]
+                    )
+            spec = make_pod_spec(zookeeper_uri)
+            log("set pod spec:\n{}".format(spec))
+            pod_spec_set(spec)
+            set_flag("kafka-k8s.configured")
+    except Exception as e:
+        layer.status.blocked("k8s spec failed to deploy: {}".format(e))
+
+
+@when("kafka-k8s.configured")
+def set_kafka_active():
+    layer.status.active("ready")
+
+
+@when("zookeeper.ready")
+@when_not("leadership.is_leader")
+def non_leaders_active():
+    layer.status.active("ready")
+
+
+@when("kafka-k8s.configured")
+@when("kafka.joined", "zookeeper.ready")
+def serve_client():
+    layer.status.maintenance("Sending kafka configuration")
+    try:
+        kafka = endpoint_from_flag("kafka.joined")
+        zookeeper = endpoint_from_flag("zookeeper.ready")
+        if zookeeper and kafka:
+            service_ip = get_service_ip("kafka")
+            if service_ip:
+                kafka.send_connection(
+                    get_kafka_port(), service_ip,
+                )
+                kafka.send_zookeepers(zookeeper.zookeepers())
+                clear_flag("kafka.joined")
+    except Exception as e:
+        log("Fail sending kafka configuration: {}".format(e))
+
+
+def make_pod_spec(zookeeper_uri):
+    """Make pod specification for Kubernetes
+
+    Args:
+        zookeeper_uri (str): Zookeeper hosts appended by comma.
+    Returns:
+        pod_spec: Pod specification for Kubernetes
+    """
+
+    with open("reactive/spec_template.yaml") as spec_file:
+        pod_spec_template = spec_file.read()
+
+    md = metadata()
+    cfg = config()
+
+    data = {
+        "name": md.get("name"),
+        "docker_image": cfg.get("image"),
+        "port": get_kafka_port(),
+        "zookeeper_uri": zookeeper_uri,
+    }
+    data.update(cfg)
+    return pod_spec_template % data
+
+
+def get_kafka_port():
+    """Returns Kafka port"""
+    cfg = config()
+    return cfg.get("advertised-port")
diff --git a/installers/charm/kafka-k8s/reactive/spec_template.yaml b/installers/charm/kafka-k8s/reactive/spec_template.yaml
new file mode 100644 (file)
index 0000000..f083084
--- /dev/null
@@ -0,0 +1,140 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+version: 2
+containers:
+  - name: %(name)s
+    image: %(docker_image)s
+    ports:
+      - containerPort: %(advertised-port)s
+        protocol: TCP
+    config:
+      ALLOW_ANONYMOUS_LOGIN: 'yes'
+      ENABLE_AUTO_EXTEND: "true"
+      KAFKA_ADVERTISED_HOST_NAME: %(advertised-hostname)s
+      KAFKA_ADVERTISED_PORT: %(advertised-port)s
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_RESERVED_BROKER_MAX_ID: "999999999"
+    command:
+      - sh
+      - -c
+      - "exec kafka-server-start.sh /opt/kafka/config/server.properties --override broker.id=${HOSTNAME##*-} \
+        --override listeners=PLAINTEXT://:%(advertised-port)s \
+        --override zookeeper.connect=%(zookeeper_uri)s \
+        --override log.dir=/var/lib/kafka \
+        --override auto.create.topics.enable=true \
+        --override auto.leader.rebalance.enable=true \
+        --override background.threads=10 \
+        --override compression.type=producer \
+        --override delete.topic.enable=false \
+        --override leader.imbalance.check.interval.seconds=300 \
+        --override leader.imbalance.per.broker.percentage=10 \
+        --override log.flush.interval.messages=9223372036854775807 \
+        --override log.flush.offset.checkpoint.interval.ms=60000 \
+        --override log.flush.scheduler.interval.ms=9223372036854775807 \
+        --override log.retention.bytes=-1 \
+        --override log.retention.hours=168 \
+        --override log.roll.hours=168 \
+        --override log.roll.jitter.hours=0 \
+        --override log.segment.bytes=1073741824 \
+        --override log.segment.delete.delay.ms=60000 \
+        --override message.max.bytes=1000012 \
+        --override min.insync.replicas=1 \
+        --override num.io.threads=8 \
+        --override num.network.threads=%(kafka-units)s \
+        --override num.recovery.threads.per.data.dir=1 \
+        --override num.replica.fetchers=1 \
+        --override offset.metadata.max.bytes=4096 \
+        --override offsets.commit.required.acks=-1 \
+        --override offsets.commit.timeout.ms=5000 \
+        --override offsets.load.buffer.size=5242880 \
+        --override offsets.retention.check.interval.ms=600000 \
+        --override offsets.retention.minutes=1440 \
+        --override offsets.topic.compression.codec=0 \
+        --override offsets.topic.num.partitions=50 \
+        --override offsets.topic.replication.factor=%(kafka-units)s \
+        --override offsets.topic.segment.bytes=104857600 \
+        --override queued.max.requests=500 \
+        --override quota.consumer.default=9223372036854775807 \
+        --override quota.producer.default=9223372036854775807 \
+        --override replica.fetch.min.bytes=1 \
+        --override replica.fetch.wait.max.ms=500 \
+        --override replica.high.watermark.checkpoint.interval.ms=5000 \
+        --override replica.lag.time.max.ms=10000 \
+        --override replica.socket.receive.buffer.bytes=65536 \
+        --override replica.socket.timeout.ms=30000 \
+        --override request.timeout.ms=30000 \
+        --override socket.receive.buffer.bytes=102400 \
+        --override socket.request.max.bytes=104857600 \
+        --override socket.send.buffer.bytes=102400 \
+        --override unclean.leader.election.enable=true \
+        --override zookeeper.session.timeout.ms=6000 \
+        --override zookeeper.set.acl=false \
+        --override broker.id.generation.enable=true \
+        --override connections.max.idle.ms=600000 \
+        --override controlled.shutdown.enable=true \
+        --override controlled.shutdown.max.retries=3 \
+        --override controlled.shutdown.retry.backoff.ms=5000 \
+        --override controller.socket.timeout.ms=30000 \
+        --override default.replication.factor=1 \
+        --override fetch.purgatory.purge.interval.requests=1000 \
+        --override group.max.session.timeout.ms=300000 \
+        --override group.min.session.timeout.ms=6000 \
+        --override log.cleaner.backoff.ms=15000 \
+        --override log.cleaner.dedupe.buffer.size=134217728 \
+        --override log.cleaner.delete.retention.ms=86400000 \
+        --override log.cleaner.enable=true \
+        --override log.cleaner.io.buffer.load.factor=0.9 \
+        --override log.cleaner.io.buffer.size=524288 \
+        --override log.cleaner.io.max.bytes.per.second=1.7976931348623157E308 \
+        --override log.cleaner.min.cleanable.ratio=0.5 \
+        --override log.cleaner.min.compaction.lag.ms=0 \
+        --override log.cleaner.threads=1 \
+        --override log.cleanup.policy=delete \
+        --override log.index.interval.bytes=4096 \
+        --override log.index.size.max.bytes=10485760 \
+        --override log.message.timestamp.difference.max.ms=9223372036854775807 \
+        --override log.message.timestamp.type=CreateTime \
+        --override log.preallocate=false \
+        --override log.retention.check.interval.ms=300000 \
+        --override max.connections.per.ip=2147483647 \
+        --override num.partitions=%(num-partitions)s \
+        --override producer.purgatory.purge.interval.requests=1000 \
+        --override replica.fetch.backoff.ms=1000 \
+        --override replica.fetch.max.bytes=1048576 \
+        --override replica.fetch.response.max.bytes=10485760 \
+        --override reserved.broker.max.id=1000 "
+    kubernetes:
+      readinessProbe:
+        tcpSocket:
+          port: %(advertised-port)s
+        timeoutSeconds: 5
+        periodSeconds: 5
+        initialDelaySeconds: 10
+      livenessProbe:
+        exec:
+          command: 
+          - sh 
+          - -c 
+          - "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server=localhost:%(advertised-port)s"
+        initialDelaySeconds: 60
+        timeoutSeconds: 10
+        periodSeconds: 5
diff --git a/installers/charm/kafka-k8s/test-requirements.txt b/installers/charm/kafka-k8s/test-requirements.txt
new file mode 100644 (file)
index 0000000..b302c2e
--- /dev/null
@@ -0,0 +1,22 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+git+https://github.com/davigar15/zaza.git#egg=zaza
diff --git a/installers/charm/kafka-k8s/tests/basic_deployment.py b/installers/charm/kafka-k8s/tests/basic_deployment.py
new file mode 100644 (file)
index 0000000..363089d
--- /dev/null
@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+import unittest
+
+# import zaza.model as model
+
+
+class BasicDeployment(unittest.TestCase):
+    def test_kafka(self):
+        pass
diff --git a/installers/charm/kafka-k8s/tests/bundles/kafka-ha.yaml b/installers/charm/kafka-k8s/tests/bundles/kafka-ha.yaml
new file mode 100644 (file)
index 0000000..7c70e41
--- /dev/null
@@ -0,0 +1,44 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+bundle: kubernetes
+applications:
+  kafka-k8s:
+    charm: '../../release/'
+    scale: 2
+    options:
+      zookeeper-units: 2
+      kafka-units: 2
+    series: kubernetes
+    storage:
+      database: 50M
+  zookeeper-k8s:
+    charm: 'cs:~charmed-osm/zookeeper-k8s'
+    channel: edge
+    scale: 2
+    options:
+      zookeeper-units: 2
+    series: kubernetes
+    storage:
+      database: 50M
+relations:
+  - - 'kafka-k8s:zookeeper'
+    - 'zookeeper-k8s:zookeeper'
diff --git a/installers/charm/kafka-k8s/tests/bundles/kafka.yaml b/installers/charm/kafka-k8s/tests/bundles/kafka.yaml
new file mode 100644 (file)
index 0000000..99fd49b
--- /dev/null
@@ -0,0 +1,44 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+bundle: kubernetes
+applications:
+  kafka-k8s:
+    charm: '../../release/'
+    scale: 1
+    options:
+      zookeeper-units: 1
+      kafka-units: 1
+    series: kubernetes
+    storage:
+      database: 50M
+  zookeeper-k8s:
+    charm: 'cs:~charmed-osm/zookeeper-k8s'
+    channel: edge
+    scale: 1
+    options:
+      zookeeper-units: 1
+    series: kubernetes
+    storage:
+      database: 50M
+relations:
+  - - 'kafka-k8s:zookeeper'
+    - 'zookeeper-k8s:zookeeper'
diff --git a/installers/charm/kafka-k8s/tests/tests.yaml b/installers/charm/kafka-k8s/tests/tests.yaml
new file mode 100644 (file)
index 0000000..c90cd25
--- /dev/null
@@ -0,0 +1,28 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+gate_bundles:
+  - kafka
+  - kafka-ha
+smoke_bundles:
+  - kafka
+tests:
+  - tests.basic_deployment.BasicDeployment
diff --git a/installers/charm/kafka-k8s/tox.ini b/installers/charm/kafka-k8s/tox.ini
new file mode 100644 (file)
index 0000000..a621d6a
--- /dev/null
@@ -0,0 +1,85 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: legal@canonical.com
+#
+# To get in touch with the maintainers, please contact:
+# osm-charmers@lists.launchpad.net
+##
+
+[tox]
+envlist = pep8
+skipsdist = True
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONHASHSEED=0
+whitelist_externals = juju
+passenv = HOME TERM CS_API_* OS_* AMULET_*
+deps = -r{toxinidir}/test-requirements.txt
+install_command =
+  pip install {opts} {packages}
+
+[testenv:build]
+basepython = python3
+passenv=HTTP_PROXY HTTPS_PROXY NO_PROXY
+setenv = CHARM_LAYERS_DIR = /tmp
+         CHARM_INTERFACES_DIR = /tmp/canonical-osm/charms/interfaces/
+whitelist_externals = git
+                      charm
+                      mkdir
+                      rm
+                      mv
+                      cp
+commands =
+    rm -rf /tmp/canonical-osm /tmp/osm-common
+    rm -rf release/
+    git clone https://git.launchpad.net/canonical-osm /tmp/canonical-osm
+    git clone https://git.launchpad.net/charm-osm-common /tmp/osm-common
+    charm build . --build-dir /tmp
+    mv /tmp/kafka-k8s/ release/
+
+[testenv:black]
+basepython = python3
+deps =
+    black
+    yamllint
+    flake8
+commands =
+    black --check --diff .
+    yamllint .
+    flake8 reactive/ --max-line-length=88
+    flake8 tests/ --max-line-length=88
+[testenv:pep8]
+basepython = python3
+deps=charm-tools
+commands = charm-proof
+
+[testenv:func-noop]
+basepython = python3
+commands =
+    true
+
+[testenv:func]
+basepython = python3
+commands = functest-run-suite
+
+
+[testenv:func-smoke]
+basepython = python3
+commands = functest-run-suite --keep-model --smoke
+
+[testenv:venv]
+commands = {posargs}