From: David Garcia Date: Mon, 21 Feb 2022 10:48:11 +0000 (+0100) Subject: CharmHub and new kafka and zookeeper charms X-Git-Tag: v12.0.0rc1~35 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=4a0db7c233b71d100e3db32bb15e8aa720c9034c;p=osm%2Fdevops.git CharmHub and new kafka and zookeeper charms - Charmed installer uses bundles published in CharmHub - Use new zookeeper and kafka sidecar-charm - Changes in the charms to integrate with the new Kafka Change-Id: Ie59fe1c7c72774b317abe2433fafb28a11472b72 Signed-off-by: David Garcia --- diff --git a/installers/charm/build.sh b/installers/charm/build.sh index 65dd87da..459da132 100755 --- a/installers/charm/build.sh +++ b/installers/charm/build.sh @@ -17,12 +17,12 @@ function build() { cd $1 && tox -qe build && cd .. } -charms="ro nbi pla pol mon lcm ng-ui keystone grafana prometheus mariadb-k8s mongodb-k8s zookeeper-k8s kafka-k8s mongodb-exporter kafka-exporter mysqld-exporter" +charms="ro nbi pla pol mon lcm ng-ui grafana prometheus mongodb-exporter kafka-exporter mysqld-exporter" if [ -z `which charmcraft` ]; then - sudo snap install charmcraft --edge + sudo snap install charmcraft --classic fi for charm_directory in $charms; do - build $charm_directory & + build $charm_directory done wait \ No newline at end of file diff --git a/installers/charm/bundles/.gitignore b/installers/charm/bundles/.gitignore new file mode 100644 index 00000000..00b9f63e --- /dev/null +++ b/installers/charm/bundles/.gitignore @@ -0,0 +1,17 @@ +# Copyright 2022 ETSI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*.zip +*build/ \ No newline at end of file diff --git a/installers/charm/bundles/osm-ha/bundle.yaml b/installers/charm/bundles/osm-ha/bundle.yaml index 202ee2b7..f20e598a 100644 --- a/installers/charm/bundles/osm-ha/bundle.yaml +++ b/installers/charm/bundles/osm-ha/bundle.yaml @@ -30,17 +30,24 @@ description: | - Availability of managed services applications: zookeeper: - charm: osm-zookeeper + charm: zookeeper-k8s channel: latest/edge scale: 3 - series: kubernetes storage: - database: 100M - options: - zookeeper-units: 3 + data: 100M annotations: gui-x: 0 gui-y: 500 + kafka: + charm: kafka-k8s + channel: latest/edge + scale: 3 + trust: true + storage: + data: 100M + annotations: + gui-x: 0 + gui-y: 250 mariadb: charm: charmed-osm-mariadb-k8s scale: 3 @@ -55,19 +62,6 @@ applications: annotations: gui-x: -300 gui-y: -250 - kafka: - charm: osm-kafka - channel: latest/edge - scale: 3 - series: kubernetes - storage: - database: 100M - options: - zookeeper-units: 3 - kafka-units: 3 - annotations: - gui-x: 0 - gui-y: 250 mongodb: charm: mongodb-k8s channel: latest/stable @@ -87,6 +81,8 @@ applications: database_commonkey: osm auth_backend: keystone log_level: DEBUG + resources: + image: opensourcemano/nbi:testing-daily annotations: gui-x: 0 gui-y: -250 @@ -97,6 +93,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/ro:testing-daily annotations: gui-x: -300 gui-y: 250 @@ -105,6 +103,8 @@ applications: channel: latest/edge scale: 3 series: kubernetes + resources: + image: opensourcemano/ng-ui:testing-daily annotations: gui-x: 600 gui-y: 0 @@ -116,6 +116,8 @@ applications: options: database_commonkey: osm log_level: DEBUG + resources: + image: opensourcemano/lcm:testing-daily annotations: gui-x: -300 gui-y: 0 @@ -128,6 +130,8 @@ applications: database_commonkey: osm log_level: DEBUG keystone_enabled: true + resources: + image: opensourcemano/mon:testing-daily annotations: gui-x: 300 gui-y: 0 @@ -138,6 +142,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/pol:testing-daily annotations: gui-x: -300 gui-y: 500 @@ -148,6 +154,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/pla:testing-daily annotations: gui-x: 600 gui-y: -250 @@ -175,6 +183,8 @@ applications: charm: osm-keystone channel: latest/edge scale: 1 + resources: + keystone-image: opensourcemano/keystone:testing-daily annotations: gui-x: 300 gui-y: -250 diff --git a/installers/charm/bundles/osm/bundle.yaml b/installers/charm/bundles/osm/bundle.yaml index fa367dc5..601e3657 100644 --- a/installers/charm/bundles/osm/bundle.yaml +++ b/installers/charm/bundles/osm/bundle.yaml @@ -29,15 +29,24 @@ description: | - Availability of managed services applications: zookeeper: - charm: osm-zookeeper + charm: zookeeper-k8s channel: latest/edge scale: 1 - series: kubernetes storage: - database: 100M + data: 100M annotations: gui-x: 0 gui-y: 500 + kafka: + charm: kafka-k8s + channel: latest/edge + scale: 1 + trust: true + storage: + data: 100M + annotations: + gui-x: 0 + gui-y: 250 mariadb: charm: charmed-osm-mariadb-k8s scale: 1 @@ -51,16 +60,6 @@ applications: annotations: gui-x: -300 gui-y: -250 - kafka: - charm: osm-kafka - channel: latest/edge - scale: 1 - series: kubernetes - storage: - database: 100M - annotations: - gui-x: 0 - gui-y: 250 mongodb: charm: mongodb-k8s channel: latest/stable @@ -80,6 +79,8 @@ applications: database_commonkey: osm auth_backend: keystone log_level: DEBUG + resources: + image: opensourcemano/nbi:testing-daily annotations: gui-x: 0 gui-y: -250 @@ -90,6 +91,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/ro:testing-daily annotations: gui-x: -300 gui-y: 250 @@ -98,6 +101,8 @@ applications: channel: latest/edge scale: 1 series: kubernetes + resources: + image: opensourcemano/ng-ui:testing-daily annotations: gui-x: 600 gui-y: 0 @@ -109,6 +114,8 @@ applications: options: database_commonkey: osm log_level: DEBUG + resources: + image: opensourcemano/lcm:testing-daily annotations: gui-x: -300 gui-y: 0 @@ -121,6 +128,8 @@ applications: database_commonkey: osm log_level: DEBUG keystone_enabled: true + resources: + image: opensourcemano/mon:testing-daily annotations: gui-x: 300 gui-y: 0 @@ -131,6 +140,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/pol:testing-daily annotations: gui-x: -300 gui-y: 500 @@ -141,6 +152,8 @@ applications: series: kubernetes options: log_level: DEBUG + resources: + image: opensourcemano/pla:testing-daily annotations: gui-x: 600 gui-y: -250 @@ -168,6 +181,8 @@ applications: charm: osm-keystone channel: latest/edge scale: 1 + resources: + keystone-image: opensourcemano/keystone:testing-daily annotations: gui-x: 300 gui-y: -250 diff --git a/installers/charm/grafana/tox.ini b/installers/charm/grafana/tox.ini index ab5c86d9..58e13a66 100644 --- a/installers/charm/grafana/tox.ini +++ b/installers/charm/grafana/tox.ini @@ -99,7 +99,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/kafka-exporter/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/kafka-exporter/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/kafka-exporter/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/kafka-exporter/src/charm.py b/installers/charm/kafka-exporter/src/charm.py index 1316a4df..e6b2bf76 100755 --- a/installers/charm/kafka-exporter/src/charm.py +++ b/installers/charm/kafka-exporter/src/charm.py @@ -28,10 +28,10 @@ from pathlib import Path from typing import NoReturn, Optional from urllib.parse import urlparse +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing from opslib.osm.interfaces.grafana import GrafanaDashboardTarget -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.prometheus import PrometheusScrapeTarget from opslib.osm.pod import ( ContainerV3Builder, @@ -83,13 +83,16 @@ class ConfigModel(ModelValidator): class KafkaExporterCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__(*args, oci_image="image") # Provision Kafka relation to exchange information - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) # Register relation to provide a Scraping Target self.scrape_target = PrometheusScrapeTarget(self, "prometheus-scrape") @@ -152,10 +155,7 @@ class KafkaExporterCharm(CharmedOsmBase): """ missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if missing_relations: @@ -208,7 +208,7 @@ class KafkaExporterCharm(CharmedOsmBase): container_builder.add_command( [ "kafka_exporter", - f"--kafka.server={self.kafka_client.host}:{self.kafka_client.port}", + f"--kafka.server={self.kafka.host}:{self.kafka.port}", ] ) container = container_builder.build() diff --git a/installers/charm/kafka-exporter/tests/test_charm.py b/installers/charm/kafka-exporter/tests/test_charm.py index 3f266fe9..c00943b8 100644 --- a/installers/charm/kafka-exporter/tests/test_charm.py +++ b/installers/charm/kafka-exporter/tests/test_charm.py @@ -87,7 +87,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) diff --git a/installers/charm/kafka-exporter/tox.ini b/installers/charm/kafka-exporter/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/kafka-exporter/tox.ini +++ b/installers/charm/kafka-exporter/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/keystone/tox.ini b/installers/charm/keystone/tox.ini index a959d361..88fd16a2 100644 --- a/installers/charm/keystone/tox.ini +++ b/installers/charm/keystone/tox.ini @@ -99,7 +99,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/lcm/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/lcm/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/lcm/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/lcm/src/charm.py b/installers/charm/lcm/src/charm.py index 4a10f5c7..16e6f89f 100755 --- a/installers/charm/lcm/src/charm.py +++ b/installers/charm/lcm/src/charm.py @@ -27,10 +27,10 @@ import logging from typing import NoReturn, Optional +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing from opslib.osm.interfaces.http import HttpClient -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.pod import ContainerV3Builder, PodRestartPolicy, PodSpecV3Builder from opslib.osm.validator import ModelValidator, validator @@ -140,6 +140,9 @@ class ConfigModel(ModelValidator): class LcmCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__( *args, @@ -164,9 +167,9 @@ class LcmCharm(CharmedOsmBase): }, }, ) - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mongodb_client = MongoClient(self, "mongodb") self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) @@ -179,10 +182,7 @@ class LcmCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -241,8 +241,8 @@ class LcmCharm(CharmedOsmBase): "OSMLCM_RO_TENANT": "osm", # Kafka configuration "OSMLCM_MESSAGE_DRIVER": "kafka", - "OSMLCM_MESSAGE_HOST": self.kafka_client.host, - "OSMLCM_MESSAGE_PORT": self.kafka_client.port, + "OSMLCM_MESSAGE_HOST": self.kafka.host, + "OSMLCM_MESSAGE_PORT": self.kafka.port, # Database configuration "OSMLCM_DATABASE_DRIVER": "mongo", # Storage configuration diff --git a/installers/charm/lcm/tests/test_charm.py b/installers/charm/lcm/tests/test_charm.py index 776b3845..aa11a747 100644 --- a/installers/charm/lcm/tests/test_charm.py +++ b/installers/charm/lcm/tests/test_charm.py @@ -217,7 +217,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) def initialize_mongo_config(self): diff --git a/installers/charm/lcm/tox.ini b/installers/charm/lcm/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/lcm/tox.ini +++ b/installers/charm/lcm/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/local_osm_bundle.yaml b/installers/charm/local_osm_bundle.yaml index 216718de..6ab0df6b 100644 --- a/installers/charm/local_osm_bundle.yaml +++ b/installers/charm/local_osm_bundle.yaml @@ -11,23 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -description: Single instance OSM bundle +name: osm bundle: kubernetes +description: Local bundle for development applications: zookeeper: - charm: "./zookeeper/zookeeper.charm" + charm: zookeeper-k8s + channel: latest/edge scale: 1 - series: kubernetes storage: - database: 100M - resources: - image: rocks.canonical.com:443/k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10 + data: 100M annotations: gui-x: 0 - gui-y: 550 - mariadb-k8s: - charm: "cs:~charmed-osm/mariadb-k8s" - channel: "stable" + gui-y: 500 + mariadb: + charm: charmed-osm-mariadb-k8s scale: 1 series: kubernetes storage: @@ -37,21 +35,21 @@ applications: root_password: osm4u user: mano annotations: - gui-x: -250 - gui-y: -200 + gui-x: -300 + gui-y: -250 kafka: - charm: "./kafka/kafka.charm" + charm: kafka-k8s + channel: latest/edge scale: 1 - series: kubernetes + trust: true storage: - database: 100M - resources: - image: rocks.canonical.com:443/wurstmeister/kafka:2.12-2.2.1 + data: 100M annotations: gui-x: 0 - gui-y: 300 + gui-y: 250 mongodb: - charm: ch:mongodb-k8s + charm: mongodb-k8s + channel: latest/stable scale: 1 series: kubernetes storage: @@ -60,116 +58,121 @@ applications: gui-x: 0 gui-y: 0 nbi: - charm: "./nbi/nbi.charm" + charm: ./nbi/osm-nbi.charm scale: 1 + resources: + image: opensourcemano/nbi:testing-daily series: kubernetes options: database_commonkey: osm auth_backend: keystone - resources: - image: opensourcemano/nbi:testing-daily + log_level: DEBUG annotations: gui-x: 0 - gui-y: -200 + gui-y: -250 ro: - charm: "./ro/ro.charm" + charm: ./ro/osm-ro.charm scale: 1 - series: kubernetes resources: image: opensourcemano/ro:testing-daily + series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 300 + gui-x: -300 + gui-y: 250 ng-ui: - charm: "./ng-ui/ng-ui.charm" + charm: ./ng-ui/osm-ng-ui.charm scale: 1 - series: kubernetes resources: image: opensourcemano/ng-ui:testing-daily + series: kubernetes annotations: - gui-x: 500 - gui-y: 100 + gui-x: 600 + gui-y: 0 lcm: - charm: "./lcm/lcm.charm" + charm: ./lcm/osm-lcm.charm scale: 1 + resources: + image: opensourcemano/lcm:testing-daily series: kubernetes options: database_commonkey: osm - resources: - image: opensourcemano/lcm:testing-daily + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 50 + gui-x: -300 + gui-y: 0 mon: - charm: "./mon/mon.charm" + charm: ./mon/osm-mon.charm scale: 1 + resources: + image: opensourcemano/mon:testing-daily series: kubernetes options: database_commonkey: osm - resources: - image: opensourcemano/mon:testing-daily + log_level: DEBUG + keystone_enabled: true annotations: - gui-x: 250 - gui-y: 50 + gui-x: 300 + gui-y: 0 pol: - charm: "./pol/pol.charm" + charm: ./pol/osm-pol.charm scale: 1 - series: kubernetes resources: image: opensourcemano/pol:testing-daily + series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 550 + gui-x: -300 + gui-y: 500 pla: - charm: "./pla/pla.charm" + charm: ./pla/osm-pla.charm scale: 1 - series: kubernetes resources: image: opensourcemano/pla:testing-daily + series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: 500 - gui-y: -200 + gui-x: 600 + gui-y: -250 prometheus: - charm: "./prometheus/prometheus.charm" - channel: "stable" + charm: osm-prometheus + channel: latest/edge scale: 1 series: kubernetes storage: data: 50M options: default-target: "mon:8000" - resources: - image: ubuntu/prometheus:latest - backup-image: ed1000/prometheus-backup:latest annotations: - gui-x: 250 - gui-y: 300 + gui-x: 300 + gui-y: 250 grafana: - charm: "./grafana/grafana.charm" - channel: "stable" + charm: osm-grafana + channel: latest/edge scale: 1 series: kubernetes - resources: - image: ubuntu/grafana:latest annotations: - gui-x: 250 - gui-y: 550 + gui-x: 300 + gui-y: 500 keystone: - charm: "./keystone/keystone.charm" + charm: osm-keystone + channel: latest/edge resources: - image: opensourcemano/keystone:testing-daily + keystone-image: opensourcemano/keystone:testing-daily scale: 1 - series: kubernetes annotations: - gui-x: -250 - gui-y: 550 + gui-x: 300 + gui-y: -250 relations: - - grafana:prometheus - prometheus:prometheus - - kafka:zookeeper - zookeeper:zookeeper - - keystone:db - - mariadb-k8s:mysql + - mariadb:mysql - - lcm:kafka - kafka:kafka - - lcm:mongodb @@ -206,7 +209,7 @@ relations: - nbi:nbi - - mon:keystone - keystone:keystone - - - mariadb-k8s:mysql + - - mariadb:mysql - pol:mysql - - - mariadb-k8s:mysql - - grafana:db + - - grafana:db + - mariadb:mysql diff --git a/installers/charm/local_osm_ha_bundle.yaml b/installers/charm/local_osm_ha_bundle.yaml index 0a08eaa8..79950cad 100644 --- a/installers/charm/local_osm_ha_bundle.yaml +++ b/installers/charm/local_osm_ha_bundle.yaml @@ -11,24 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -description: A high-available OSM cluster. +name: osm-ha bundle: kubernetes +description: Local bundle for development (HA) applications: - zookeeper-k8s: - charm: "cs:~charmed-osm/zookeeper-k8s" - channel: "stable" + zookeeper: + charm: zookeeper-k8s + channel: latest/edge scale: 3 - series: kubernetes storage: - database: 100M - options: - zookeeper-units: 3 + data: 100M annotations: gui-x: 0 - gui-y: 550 - mariadb-k8s: - charm: "cs:~charmed-osm/mariadb-k8s" - channel: "stable" + gui-y: 500 + mariadb: + charm: charmed-osm-mariadb-k8s scale: 3 series: kubernetes storage: @@ -39,151 +36,170 @@ applications: user: mano ha-mode: true annotations: - gui-x: -250 - gui-y: -200 - kafka-k8s: - charm: "cs:~charmed-osm/kafka-k8s" - channel: "stable" + gui-x: -300 + gui-y: -250 + kafka: + charm: kafka-k8s + channel: latest/edge scale: 3 - series: kubernetes + trust: true storage: - database: 100M - options: - zookeeper-units: 3 - kafka-units: 3 + data: 100M annotations: gui-x: 0 - gui-y: 300 - mongodb-k8s: - charm: "cs:~charmed-osm/mongodb-k8s" - channel: "stable" + gui-y: 250 + mongodb: + charm: mongodb-k8s + channel: latest/stable scale: 3 series: kubernetes storage: - database: 50M - options: - replica-set: rs0 - namespace: osm - enable-sidecar: true + db: 50M annotations: gui-x: 0 - gui-y: 50 + gui-y: 0 nbi: - charm: "./nbi/build" + charm: ./nbi/osm-nbi.charm scale: 3 + resources: + image: opensourcemano/nbi:testing-daily series: kubernetes options: database_commonkey: osm auth_backend: keystone + log_level: DEBUG annotations: gui-x: 0 - gui-y: -200 + gui-y: -250 ro: - charm: "./ro/build" + charm: ./ro/osm-ro.charm scale: 3 + resources: + image: opensourcemano/ro:testing-daily series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 300 + gui-x: -300 + gui-y: 250 ng-ui: - charm: "./ng-ui/build" + charm: ./ng-ui/osm-ng-ui.charm scale: 3 + resources: + image: opensourcemano/ng-ui:testing-daily series: kubernetes annotations: - gui-x: 500 - gui-y: 100 + gui-x: 600 + gui-y: 0 lcm: - charm: "./lcm/build" + charm: ./lcm/osm-lcm.charm scale: 3 + resources: + image: opensourcemano/lcm:testing-daily series: kubernetes options: database_commonkey: osm + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 50 + gui-x: -300 + gui-y: 0 mon: - charm: "./mon/build" - scale: 1 + charm: ./mon/osm-mon.charm + scale: 3 + resources: + image: opensourcemano/mon:testing-daily series: kubernetes options: database_commonkey: osm + log_level: DEBUG + keystone_enabled: true annotations: - gui-x: 250 - gui-y: 50 + gui-x: 300 + gui-y: 0 pol: - charm: "./pol/build" + charm: ./pol/osm-pol.charm scale: 3 + resources: + image: opensourcemano/pol:testing-daily series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: -250 - gui-y: 550 + gui-x: -300 + gui-y: 500 pla: - charm: "./pla/build" + charm: ./pla/osm-pla.charm scale: 3 + resources: + image: opensourcemano/pla:testing-daily series: kubernetes + options: + log_level: DEBUG annotations: - gui-x: 500 - gui-y: -200 + gui-x: 600 + gui-y: -250 prometheus: - charm: "./prometheus/build" - channel: "stable" - scale: 1 + charm: osm-prometheus + channel: latest/edge + scale: 3 series: kubernetes storage: data: 50M options: default-target: "mon:8000" annotations: - gui-x: 250 - gui-y: 300 + gui-x: 300 + gui-y: 250 grafana: - charm: "./grafana/build" - channel: "stable" + charm: osm-grafana + channel: latest/edge scale: 3 series: kubernetes annotations: - gui-x: 250 - gui-y: 550 + gui-x: 300 + gui-y: 500 keystone: - charm: "./keystone/build" - scale: 3 - series: kubernetes + charm: osm-keystone + channel: latest/edge + resources: + keystone-image: opensourcemano/keystone:testing-daily + scale: 1 annotations: - gui-x: -250 - gui-y: 550 + gui-x: 300 + gui-y: -250 relations: - - grafana:prometheus - prometheus:prometheus - - - kafka-k8s:zookeeper - - zookeeper-k8s:zookeeper + - - kafka:zookeeper + - zookeeper:zookeeper - - keystone:db - - mariadb-k8s:mysql + - mariadb:mysql - - lcm:kafka - - kafka-k8s:kafka + - kafka:kafka - - lcm:mongodb - - mongodb-k8s:mongo + - mongodb:database - - ro:ro - lcm:ro - - ro:kafka - - kafka-k8s:kafka + - kafka:kafka - - ro:mongodb - - mongodb-k8s:mongo + - mongodb:database - - pol:kafka - - kafka-k8s:kafka + - kafka:kafka - - pol:mongodb - - mongodb-k8s:mongo + - mongodb:database - - mon:mongodb - - mongodb-k8s:mongo + - mongodb:database - - mon:kafka - - kafka-k8s:kafka + - kafka:kafka - - pla:kafka - - kafka-k8s:kafka + - kafka:kafka - - pla:mongodb - - mongodb-k8s:mongo + - mongodb:database - - nbi:mongodb - - mongodb-k8s:mongo + - mongodb:database - - nbi:kafka - - kafka-k8s:kafka + - kafka:kafka - - nbi:prometheus - prometheus:prometheus - - nbi:keystone @@ -192,3 +208,9 @@ relations: - prometheus:prometheus - - ng-ui:nbi - nbi:nbi + - - mon:keystone + - keystone:keystone + - - mariadb:mysql + - pol:mysql + - - grafana:db + - mariadb:mysql diff --git a/installers/charm/mon/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/mon/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/mon/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/mon/src/charm.py b/installers/charm/mon/src/charm.py index 7833deb0..d04779a0 100755 --- a/installers/charm/mon/src/charm.py +++ b/installers/charm/mon/src/charm.py @@ -28,9 +28,9 @@ import logging from typing import NoReturn, Optional +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.keystone import KeystoneClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.interfaces.prometheus import PrometheusClient @@ -125,6 +125,9 @@ class ConfigModel(ModelValidator): class MonCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__( *args, @@ -149,9 +152,9 @@ class MonCharm(CharmedOsmBase): }, }, ) - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mongodb_client = MongoClient(self, "mongodb") self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) @@ -172,10 +175,7 @@ class MonCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -270,8 +270,8 @@ class MonCharm(CharmedOsmBase): "OSMMON_EVALUATOR_INTERVAL": config.evaluator_interval, # Kafka configuration "OSMMON_MESSAGE_DRIVER": "kafka", - "OSMMON_MESSAGE_HOST": self.kafka_client.host, - "OSMMON_MESSAGE_PORT": self.kafka_client.port, + "OSMMON_MESSAGE_HOST": self.kafka.host, + "OSMMON_MESSAGE_PORT": self.kafka.port, # Database configuration "OSMMON_DATABASE_DRIVER": "mongo", # Prometheus configuration diff --git a/installers/charm/mon/tests/test_charm.py b/installers/charm/mon/tests/test_charm.py index a7cef207..e9748d30 100644 --- a/installers/charm/mon/tests/test_charm.py +++ b/installers/charm/mon/tests/test_charm.py @@ -148,7 +148,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) def initialize_mongo_config(self): diff --git a/installers/charm/mon/tox.ini b/installers/charm/mon/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/mon/tox.ini +++ b/installers/charm/mon/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/mongodb-exporter/tox.ini b/installers/charm/mongodb-exporter/tox.ini index 8e3318a4..4c7970df 100644 --- a/installers/charm/mongodb-exporter/tox.ini +++ b/installers/charm/mongodb-exporter/tox.ini @@ -99,7 +99,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/nbi/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/nbi/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/nbi/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/nbi/src/charm.py b/installers/charm/nbi/src/charm.py index faba78ab..4aaecb9d 100755 --- a/installers/charm/nbi/src/charm.py +++ b/installers/charm/nbi/src/charm.py @@ -29,10 +29,10 @@ from typing import NoReturn, Optional from urllib.parse import urlparse +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing from opslib.osm.interfaces.http import HttpServer -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.keystone import KeystoneClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.interfaces.prometheus import PrometheusClient @@ -118,6 +118,9 @@ class ConfigModel(ModelValidator): class NbiCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__( *args, @@ -139,9 +142,9 @@ class NbiCharm(CharmedOsmBase): }, ) - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mongodb_client = MongoClient(self, "mongodb") self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) @@ -174,10 +177,7 @@ class NbiCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -227,7 +227,7 @@ class NbiCharm(CharmedOsmBase): "command": [ "sh", "-c", - f"until (nc -zvw1 {self.kafka_client.host} {self.kafka_client.port} ); do sleep 3; done; exit 0", + f"until (nc -zvw1 {self.kafka.host} {self.kafka.port} ); do sleep 3; done; exit 0", ], } ) @@ -257,9 +257,9 @@ class NbiCharm(CharmedOsmBase): "OSMNBI_SERVER_ENABLE_TEST": config.enable_test, "OSMNBI_STATIC_DIR": "/app/osm_nbi/html_public", # Kafka configuration - "OSMNBI_MESSAGE_HOST": self.kafka_client.host, + "OSMNBI_MESSAGE_HOST": self.kafka.host, "OSMNBI_MESSAGE_DRIVER": "kafka", - "OSMNBI_MESSAGE_PORT": self.kafka_client.port, + "OSMNBI_MESSAGE_PORT": self.kafka.port, # Database configuration "OSMNBI_DATABASE_DRIVER": "mongo", # Storage configuration diff --git a/installers/charm/nbi/tests/test_charm.py b/installers/charm/nbi/tests/test_charm.py index 116c06bc..92c29808 100644 --- a/installers/charm/nbi/tests/test_charm.py +++ b/installers/charm/nbi/tests/test_charm.py @@ -159,7 +159,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) def initialize_mongo_config(self): diff --git a/installers/charm/nbi/tox.ini b/installers/charm/nbi/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/nbi/tox.ini +++ b/installers/charm/nbi/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/ng-ui/tox.ini b/installers/charm/ng-ui/tox.ini index ab5c86d9..58e13a66 100644 --- a/installers/charm/ng-ui/tox.ini +++ b/installers/charm/ng-ui/tox.ini @@ -99,7 +99,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/pla/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/pla/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/pla/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/pla/src/charm.py b/installers/charm/pla/src/charm.py index 3238dde5..2663763b 100755 --- a/installers/charm/pla/src/charm.py +++ b/installers/charm/pla/src/charm.py @@ -26,9 +26,9 @@ import logging from typing import NoReturn, Optional +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.pod import ( ContainerV3Builder, @@ -76,12 +76,15 @@ class ConfigModel(ModelValidator): class PlaCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__(*args, oci_image="image") - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mongodb_client = MongoClient(self, "mongodb") self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) @@ -90,10 +93,7 @@ class PlaCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -141,8 +141,8 @@ class PlaCharm(CharmedOsmBase): "OSMPLA_GLOBAL_LOG_LEVEL": config.log_level, # Kafka configuration "OSMPLA_MESSAGE_DRIVER": "kafka", - "OSMPLA_MESSAGE_HOST": self.kafka_client.host, - "OSMPLA_MESSAGE_PORT": self.kafka_client.port, + "OSMPLA_MESSAGE_HOST": self.kafka.host, + "OSMPLA_MESSAGE_PORT": self.kafka.port, # Database configuration "OSMPLA_DATABASE_DRIVER": "mongo", } diff --git a/installers/charm/pla/tests/test_charm.py b/installers/charm/pla/tests/test_charm.py index debc378b..d577e9fb 100644 --- a/installers/charm/pla/tests/test_charm.py +++ b/installers/charm/pla/tests/test_charm.py @@ -102,7 +102,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) def initialize_mongo_config(self): diff --git a/installers/charm/pla/tox.ini b/installers/charm/pla/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/pla/tox.ini +++ b/installers/charm/pla/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/pol/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/pol/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/pol/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/pol/src/charm.py b/installers/charm/pol/src/charm.py index e2fcdb3e..7b92b453 100755 --- a/installers/charm/pol/src/charm.py +++ b/installers/charm/pol/src/charm.py @@ -27,9 +27,9 @@ import logging import re from typing import NoReturn, Optional +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.interfaces.mysql import MysqlClient from opslib.osm.pod import ( @@ -87,6 +87,9 @@ class ConfigModel(ModelValidator): class PolCharm(CharmedOsmBase): + + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: super().__init__( *args, @@ -107,9 +110,9 @@ class PolCharm(CharmedOsmBase): }, }, ) - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mongodb_client = MongoClient(self, "mongodb") self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) @@ -122,10 +125,7 @@ class PolCharm(CharmedOsmBase): def _check_missing_dependencies(self, config: ConfigModel): missing_relations = [] - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -185,8 +185,8 @@ class PolCharm(CharmedOsmBase): "OSMPOL_GLOBAL_LOGLEVEL": config.log_level, # Kafka configuration "OSMPOL_MESSAGE_DRIVER": "kafka", - "OSMPOL_MESSAGE_HOST": self.kafka_client.host, - "OSMPOL_MESSAGE_PORT": self.kafka_client.port, + "OSMPOL_MESSAGE_HOST": self.kafka.host, + "OSMPOL_MESSAGE_PORT": self.kafka.port, # Database configuration "OSMPOL_DATABASE_DRIVER": "mongo", } diff --git a/installers/charm/pol/tests/test_charm.py b/installers/charm/pol/tests/test_charm.py index 330ecee6..6cf435d0 100644 --- a/installers/charm/pol/tests/test_charm.py +++ b/installers/charm/pol/tests/test_charm.py @@ -132,7 +132,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) def initialize_mongo_config(self): diff --git a/installers/charm/pol/tox.ini b/installers/charm/pol/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/pol/tox.ini +++ b/installers/charm/pol/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/prometheus/tox.ini b/installers/charm/prometheus/tox.ini index 8e3318a4..4c7970df 100644 --- a/installers/charm/prometheus/tox.ini +++ b/installers/charm/prometheus/tox.ini @@ -99,7 +99,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charm/ro/lib/charms/kafka_k8s/v0/kafka.py b/installers/charm/ro/lib/charms/kafka_k8s/v0/kafka.py new file mode 100644 index 00000000..1baf9a88 --- /dev/null +++ b/installers/charm/ro/lib/charms/kafka_k8s/v0/kafka.py @@ -0,0 +1,207 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kafka library. + +This [library](https://juju.is/docs/sdk/libraries) implements both sides of the +`kafka` [interface](https://juju.is/docs/sdk/relations). + +The *provider* side of this interface is implemented by the +[kafka-k8s Charmed Operator](https://charmhub.io/kafka-k8s). + +Any Charmed Operator that *requires* Kafka for providing its +service should implement the *requirer* side of this interface. + +In a nutshell using this library to implement a Charmed Operator *requiring* +Kafka would look like + +``` +$ charmcraft fetch-lib charms.kafka_k8s.v0.kafka +``` + +`metadata.yaml`: + +``` +requires: + kafka: + interface: kafka + limit: 1 +``` + +`src/charm.py`: + +``` +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires +from ops.charm import CharmBase + + +class MyCharm(CharmBase): + + on = KafkaEvents() + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self) + self.framework.observe( + self.on.kafka_available, + self._on_kafka_available, + ) + self.framework.observe( + self.on.kafka_broken, + self._on_kafka_broken, + ) + + def _on_kafka_available(self, event): + # Get Kafka host and port + host: str = self.kafka.host + port: int = self.kafka.port + # host => "kafka-k8s" + # port => 9092 + + def _on_kafka_broken(self, event): + # Stop service + # ... + self.unit.status = BlockedStatus("need kafka relation") +``` + +You can file bugs +[here](https://github.com/charmed-osm/kafka-k8s-operator/issues)! +""" + +from typing import Optional + +from ops.charm import CharmBase, CharmEvents +from ops.framework import EventBase, EventSource, Object + +# The unique Charmhub library identifier, never change it +from ops.model import Relation + +LIBID = "eacc8c85082347c9aae740e0220b8376" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 3 + + +KAFKA_HOST_APP_KEY = "host" +KAFKA_PORT_APP_KEY = "port" + + +class _KafkaAvailableEvent(EventBase): + """Event emitted when Kafka is available.""" + + +class _KafkaBrokenEvent(EventBase): + """Event emitted when Kafka relation is broken.""" + + +class KafkaEvents(CharmEvents): + """Kafka events. + + This class defines the events that Kafka can emit. + + Events: + kafka_available (_KafkaAvailableEvent) + """ + + kafka_available = EventSource(_KafkaAvailableEvent) + kafka_broken = EventSource(_KafkaBrokenEvent) + + +class KafkaRequires(Object): + """Requires-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self.charm = charm + self._endpoint_name = endpoint_name + + # Observe relation events + event_observe_mapping = { + charm.on[self._endpoint_name].relation_changed: self._on_relation_changed, + charm.on[self._endpoint_name].relation_broken: self._on_relation_broken, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + def _on_relation_changed(self, event) -> None: + if event.relation.app and all( + key in event.relation.data[event.relation.app] + for key in (KAFKA_HOST_APP_KEY, KAFKA_PORT_APP_KEY) + ): + self.charm.on.kafka_available.emit() + + def _on_relation_broken(self, _) -> None: + self.charm.on.kafka_broken.emit() + + @property + def host(self) -> str: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + relation.data[relation.app].get(KAFKA_HOST_APP_KEY) + if relation and relation.app + else None + ) + + @property + def port(self) -> int: + relation: Relation = self.model.get_relation(self._endpoint_name) + return ( + int(relation.data[relation.app].get(KAFKA_PORT_APP_KEY)) + if relation and relation.app + else None + ) + + +class KafkaProvides(Object): + """Provides-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, endpoint_name: str = "kafka") -> None: + super().__init__(charm, endpoint_name) + self._endpoint_name = endpoint_name + + def set_host_info(self, host: str, port: int, relation: Optional[Relation] = None) -> None: + """Set Kafka host and port. + + This function writes in the application data of the relation, therefore, + only the unit leader can call it. + + Args: + host (str): Kafka hostname or IP address. + port (int): Kafka port. + relation (Optional[Relation]): Relation to update. + If not specified, all relations will be updated. + + Raises: + Exception: if a non-leader unit calls this function. + """ + if not self.model.unit.is_leader(): + raise Exception("only the leader set host information.") + + if relation: + self._update_relation_data(host, port, relation) + return + + for relation in self.model.relations[self._endpoint_name]: + self._update_relation_data(host, port, relation) + + def _update_relation_data(self, host: str, port: int, relation: Relation) -> None: + """Update data in relation if needed.""" + relation.data[self.model.app][KAFKA_HOST_APP_KEY] = host + relation.data[self.model.app][KAFKA_PORT_APP_KEY] = str(port) diff --git a/installers/charm/ro/src/charm.py b/installers/charm/ro/src/charm.py index 1367a440..b196b195 100755 --- a/installers/charm/ro/src/charm.py +++ b/installers/charm/ro/src/charm.py @@ -26,9 +26,9 @@ import base64 import logging from typing import Dict, NoReturn, Optional +from charms.kafka_k8s.v0.kafka import KafkaEvents, KafkaRequires from ops.main import main from opslib.osm.charm import CharmedOsmBase, RelationsMissing -from opslib.osm.interfaces.kafka import KafkaClient from opslib.osm.interfaces.mongo import MongoClient from opslib.osm.interfaces.mysql import MysqlClient from opslib.osm.pod import ( @@ -126,6 +126,8 @@ class ConfigModel(ModelValidator): class RoCharm(CharmedOsmBase): """GrafanaCharm Charm.""" + on = KafkaEvents() + def __init__(self, *args) -> NoReturn: """Prometheus Charm constructor.""" super().__init__( @@ -144,9 +146,9 @@ class RoCharm(CharmedOsmBase): **_get_ro_host_paths(self.config.get("debug_ro_local_path")), }, ) - self.kafka_client = KafkaClient(self, "kafka") - self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) - self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) + self.kafka = KafkaRequires(self) + self.framework.observe(self.on.kafka_available, self.configure_pod) + self.framework.observe(self.on.kafka_broken, self.configure_pod) self.mysql_client = MysqlClient(self, "mysql") self.framework.observe(self.on["mysql"].relation_changed, self.configure_pod) @@ -176,10 +178,7 @@ class RoCharm(CharmedOsmBase): missing_relations = [] if config.enable_ng_ro: - if ( - self.kafka_client.is_missing_data_in_unit() - and self.kafka_client.is_missing_data_in_app() - ): + if not self.kafka.host or not self.kafka.port: missing_relations.append("kafka") if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): missing_relations.append("mongodb") @@ -286,8 +285,8 @@ class RoCharm(CharmedOsmBase): container_builder.add_envs( { "OSMRO_MESSAGE_DRIVER": "kafka", - "OSMRO_MESSAGE_HOST": self.kafka_client.host, - "OSMRO_MESSAGE_PORT": self.kafka_client.port, + "OSMRO_MESSAGE_HOST": self.kafka.host, + "OSMRO_MESSAGE_PORT": self.kafka.port, # MongoDB configuration "OSMRO_DATABASE_DRIVER": "mongo", } diff --git a/installers/charm/ro/tests/test_charm.py b/installers/charm/ro/tests/test_charm.py index 2d317bd3..f18e7682 100644 --- a/installers/charm/ro/tests/test_charm.py +++ b/installers/charm/ro/tests/test_charm.py @@ -125,7 +125,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) # Initializing the mongodb config @@ -143,7 +143,7 @@ class TestCharm(unittest.TestCase): kafka_relation_id = self.harness.add_relation("kafka", "kafka") self.harness.add_relation_unit(kafka_relation_id, "kafka/0") self.harness.update_relation_data( - kafka_relation_id, "kafka/0", {"host": "kafka", "port": 9092} + kafka_relation_id, "kafka", {"host": "kafka", "port": 9092} ) # Initializing the mongo relation diff --git a/installers/charm/ro/tox.ini b/installers/charm/ro/tox.ini index 8e3318a4..f3c91440 100644 --- a/installers/charm/ro/tox.ini +++ b/installers/charm/ro/tox.ini @@ -29,8 +29,10 @@ toxworkdir = /tmp/.tox [testenv] basepython = python3.8 -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 +setenv = + VIRTUAL_ENV={envdir} + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{toxinidir}/src + PYTHONDONTWRITEBYTECODE = 1 deps = -r{toxinidir}/requirements.txt @@ -99,7 +101,7 @@ whitelist_externals = charmcraft sh commands = - charmcraft build + charmcraft pack sh -c 'ubuntu_version=20.04; \ architectures="amd64-aarch64-arm64"; \ charm_name=`cat metadata.yaml | grep -E "^name: " | cut -f 2 -d " "`; \ diff --git a/installers/charmed_install.sh b/installers/charmed_install.sh index a2b04aad..d26b9e73 100755 --- a/installers/charmed_install.sh +++ b/installers/charmed_install.sh @@ -27,15 +27,10 @@ PATH=/snap/bin:${PATH} MODEL_NAME=osm -# Latest bundles using old mongodb-k8s -# OSM_BUNDLE=cs:osm-68 -# OSM_HA_BUNDLE=cs:osm-ha-54 -# The charm store does not support referencing charms from CharmHub, -# therefore we will point to the local bundles until we migrate all -# charms to CharmHub. -OSM_BUNDLE=/usr/share/osm-devops/installers/charm/bundles/osm/bundle.yaml -OSM_HA_BUNDLE=/usr/share/osm-devops/installers/charm/bundles/osm-ha/bundle.yaml -TAG=testing-daily +OSM_BUNDLE=ch:osm +OSM_HA_BUNDLE=ch:osm-ha +CHARMHUB_CHANNEL=latest/edge +unset TAG function check_arguments(){ while [ $# -gt 0 ] ; do @@ -263,9 +258,9 @@ function deploy_charmed_osm(){ fi if [ -v BUNDLE ]; then - juju deploy -m $MODEL_NAME $BUNDLE --overlay ~/.osm/vca-overlay.yaml $images_overlay $extra_overlay + juju deploy --trust --channel $CHARMHUB_CHANNEL -m $MODEL_NAME $BUNDLE --overlay ~/.osm/vca-overlay.yaml $images_overlay $extra_overlay else - juju deploy -m $MODEL_NAME $OSM_BUNDLE --overlay ~/.osm/vca-overlay.yaml $images_overlay $extra_overlay + juju deploy --trust --channel $CHARMHUB_CHANNEL -m $MODEL_NAME $OSM_BUNDLE --overlay ~/.osm/vca-overlay.yaml $images_overlay $extra_overlay fi if [ ! -v KUBECFG ]; then