X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=installers%2Fcharm%2Fgrafana%2Fsrc%2Fcharm.py;h=caa0277969c9ccc3f5cdecbe06dc032a1a025cd2;hb=ee303f97b9afe276076f56220d73c6382aa65c48;hp=1920e762b851bf2a27ef67aed0b7bf902bbcf371;hpb=b17e76b6df29ad727711175c2b6830a9db984a1d;p=osm%2Fdevops.git diff --git a/installers/charm/grafana/src/charm.py b/installers/charm/grafana/src/charm.py index 1920e762..caa02779 100755 --- a/installers/charm/grafana/src/charm.py +++ b/installers/charm/grafana/src/charm.py @@ -20,162 +20,298 @@ # osm-charmers@lists.launchpad.net ## +# pylint: disable=E0213 + +from ipaddress import ip_network import logging -from typing import Dict, List, NoReturn +from pathlib import Path +import secrets +from string import Template +from typing import NoReturn, Optional +from urllib.parse import urlparse -from ops.charm import CharmBase -from ops.framework import StoredState from ops.main import main -from ops.model import ActiveStatus, Application, BlockedStatus, MaintenanceStatus, Unit -from oci_image import OCIImageResource, OCIImageResourceError +from opslib.osm.charm import CharmedOsmBase, RelationsMissing +from opslib.osm.interfaces.grafana import GrafanaCluster +from opslib.osm.interfaces.mysql import MysqlClient +from opslib.osm.interfaces.prometheus import PrometheusClient +from opslib.osm.pod import ( + ContainerV3Builder, + FilesV3Builder, + IngressResourceV3Builder, + PodRestartPolicy, + PodSpecV3Builder, +) +from opslib.osm.validator import ModelValidator, validator -from pod_spec import make_pod_spec logger = logging.getLogger(__name__) -GRAFANA_PORT = 3000 +class ConfigModel(ModelValidator): + log_level: str + port: int + admin_user: str + max_file_size: int + osm_dashboards: bool + site_url: Optional[str] + cluster_issuer: Optional[str] + ingress_class: Optional[str] + ingress_whitelist_source_range: Optional[str] + tls_secret_name: Optional[str] + image_pull_policy: str + security_context: bool + + @validator("log_level") + def validate_log_level(cls, v): + allowed_values = ("debug", "info", "warn", "error", "critical") + if v not in allowed_values: + separator = '", "' + raise ValueError( + f'incorrect value. Allowed values are "{separator.join(allowed_values)}"' + ) + return v + + @validator("max_file_size") + def validate_max_file_size(cls, v): + if v < 0: + raise ValueError("value must be equal or greater than 0") + return v + + @validator("site_url") + def validate_site_url(cls, v): + if v: + parsed = urlparse(v) + if not parsed.scheme.startswith("http"): + raise ValueError("value must start with http") + return v + + @validator("ingress_whitelist_source_range") + def validate_ingress_whitelist_source_range(cls, v): + if v: + ip_network(v) + return v + + @validator("image_pull_policy") + def validate_image_pull_policy(cls, v): + values = { + "always": "Always", + "ifnotpresent": "IfNotPresent", + "never": "Never", + } + v = v.lower() + if v not in values.keys(): + raise ValueError("value must be always, ifnotpresent or never") + return values[v] + + +class GrafanaCharm(CharmedOsmBase): + """GrafanaCharm Charm.""" -class RelationsMissing(Exception): - def __init__(self, missing_relations: List): - self.message = "" - if missing_relations and isinstance(missing_relations, list): - self.message += f'Waiting for {", ".join(missing_relations)} relation' - if "," in self.message: - self.message += "s" + def __init__(self, *args) -> NoReturn: + """Prometheus Charm constructor.""" + super().__init__(*args, oci_image="image", mysql_uri=True) + # Initialize relation objects + self.prometheus_client = PrometheusClient(self, "prometheus") + self.grafana_cluster = GrafanaCluster(self, "cluster") + self.mysql_client = MysqlClient(self, "db") + # Observe events + event_observer_mapping = { + self.on["prometheus"].relation_changed: self.configure_pod, + self.on["prometheus"].relation_broken: self.configure_pod, + self.on["db"].relation_changed: self.configure_pod, + self.on["db"].relation_broken: self.configure_pod, + } + for event, observer in event_observer_mapping.items(): + self.framework.observe(event, observer) + + def _build_dashboard_files(self, config: ConfigModel): + files_builder = FilesV3Builder() + files_builder.add_file( + "dashboard_osm.yaml", + Path("templates/default_dashboards.yaml").read_text(), + ) + if config.osm_dashboards: + osm_dashboards_mapping = { + "kafka_exporter_dashboard.json": "templates/kafka_exporter_dashboard.json", + "mongodb_exporter_dashboard.json": "templates/mongodb_exporter_dashboard.json", + "mysql_exporter_dashboard.json": "templates/mysql_exporter_dashboard.json", + "nodes_exporter_dashboard.json": "templates/nodes_exporter_dashboard.json", + "summary_dashboard.json": "templates/summary_dashboard.json", + } + for file_name, path in osm_dashboards_mapping.items(): + files_builder.add_file(file_name, Path(path).read_text()) + return files_builder.build() + + def _build_datasources_files(self): + files_builder = FilesV3Builder() + prometheus_user = self.prometheus_client.user + prometheus_password = self.prometheus_client.password + enable_basic_auth = all([prometheus_user, prometheus_password]) + kwargs = { + "prometheus_host": self.prometheus_client.hostname, + "prometheus_port": self.prometheus_client.port, + "enable_basic_auth": enable_basic_auth, + "user": "", + "password": "", + } + if enable_basic_auth: + kwargs["user"] = f"basic_auth_user: {prometheus_user}" + kwargs[ + "password" + ] = f"secure_json_data:\n basicAuthPassword: {prometheus_password}" + files_builder.add_file( + "datasource_prometheus.yaml", + Template(Path("templates/default_datasources.yaml").read_text()).substitute( + **kwargs + ), + ) + return files_builder.build() + def _check_missing_dependencies(self, config: ConfigModel, external_db: bool): + missing_relations = [] -class RelationDefinition: - def __init__(self, relation_name: str, keys: List, source_type): - if source_type != Application and source_type != Unit: - raise TypeError( - "source_type should be ops.model.Application or ops.model.Unit" - ) - self.relation_name = relation_name - self.keys = keys - self.source_type = source_type - - -def check_missing_relation_data( - data: Dict, - expected_relations_data: List[RelationDefinition], -): - missing_relations = [] - for relation_data in expected_relations_data: - if not all( - f"{relation_data.relation_name}_{k}" in data for k in relation_data.keys - ): - missing_relations.append(relation_data.relation_name) - if missing_relations: - raise RelationsMissing(missing_relations) - - -def get_relation_data( - charm: CharmBase, - relation_data: RelationDefinition, -) -> Dict: - data = {} - relation = charm.model.get_relation(relation_data.relation_name) - if relation: - self_app_unit = ( - charm.app if relation_data.source_type == Application else charm.unit + if self.prometheus_client.is_missing_data_in_app(): + missing_relations.append("prometheus") + + if not external_db and self.mysql_client.is_missing_data_in_unit(): + missing_relations.append("db") + + if missing_relations: + raise RelationsMissing(missing_relations) + + def build_pod_spec(self, image_info, **kwargs): + # Validate config + config = ConfigModel(**dict(self.config)) + mysql_config = kwargs["mysql_config"] + if mysql_config.mysql_uri and not self.mysql_client.is_missing_data_in_unit(): + raise Exception("Mysql data cannot be provided via config and relation") + + # Check relations + external_db = True if mysql_config.mysql_uri else False + self._check_missing_dependencies(config, external_db) + + # Get initial password + admin_initial_password = self.grafana_cluster.admin_initial_password + if not admin_initial_password: + admin_initial_password = _generate_random_password() + self.grafana_cluster.set_initial_password(admin_initial_password) + + # Create Builder for the PodSpec + pod_spec_builder = PodSpecV3Builder( + enable_security_context=config.security_context ) - expected_type = relation_data.source_type - for app_unit in relation.data: - if app_unit != self_app_unit and isinstance(app_unit, expected_type): - if all(k in relation.data[app_unit] for k in relation_data.keys): - for k in relation_data.keys: - data[f"{relation_data.relation_name}_{k}"] = relation.data[ - app_unit - ].get(k) - break - return data + # Add secrets to the pod + grafana_secret_name = f"{self.app.name}-admin-secret" + pod_spec_builder.add_secret( + grafana_secret_name, + { + "admin-password": admin_initial_password, + "mysql-url": mysql_config.mysql_uri or self.mysql_client.get_uri(), + "prometheus-user": self.prometheus_client.user, + "prometheus-password": self.prometheus_client.password, + }, + ) + + # Build Container + container_builder = ContainerV3Builder( + self.app.name, + image_info, + config.image_pull_policy, + run_as_non_root=config.security_context, + ) + container_builder.add_port(name=self.app.name, port=config.port) + container_builder.add_http_readiness_probe( + "/api/health", + config.port, + initial_delay_seconds=10, + period_seconds=10, + timeout_seconds=5, + failure_threshold=3, + ) + container_builder.add_http_liveness_probe( + "/api/health", + config.port, + initial_delay_seconds=60, + timeout_seconds=30, + failure_threshold=10, + ) + container_builder.add_volume_config( + "dashboards", + "/etc/grafana/provisioning/dashboards/", + self._build_dashboard_files(config), + ) + container_builder.add_volume_config( + "datasources", + "/etc/grafana/provisioning/datasources/", + self._build_datasources_files(), + ) + container_builder.add_envs( + { + "GF_SERVER_HTTP_PORT": config.port, + "GF_LOG_LEVEL": config.log_level, + "GF_SECURITY_ADMIN_USER": config.admin_user, + } + ) + container_builder.add_secret_envs( + secret_name=grafana_secret_name, + envs={ + "GF_SECURITY_ADMIN_PASSWORD": "admin-password", + "GF_DATABASE_URL": "mysql-url", + "PROMETHEUS_USER": "prometheus-user", + "PROMETHEUS_PASSWORD": "prometheus-password", + }, + ) + container = container_builder.build() + pod_spec_builder.add_container(container) + + # Add Pod restart policy + restart_policy = PodRestartPolicy() + restart_policy.add_secrets(secret_names=(grafana_secret_name,)) + pod_spec_builder.set_restart_policy(restart_policy) + + # Add ingress resources to pod spec if site url exists + if config.site_url: + parsed = urlparse(config.site_url) + annotations = { + "nginx.ingress.kubernetes.io/proxy-body-size": "{}".format( + str(config.max_file_size) + "m" + if config.max_file_size > 0 + else config.max_file_size + ) + } + if config.ingress_class: + annotations["kubernetes.io/ingress.class"] = config.ingress_class + ingress_resource_builder = IngressResourceV3Builder( + f"{self.app.name}-ingress", annotations + ) -class GrafanaCharm(CharmBase): - """Grafana Charm.""" + if config.ingress_whitelist_source_range: + annotations[ + "nginx.ingress.kubernetes.io/whitelist-source-range" + ] = config.ingress_whitelist_source_range - state = StoredState() + if config.cluster_issuer: + annotations["cert-manager.io/cluster-issuer"] = config.cluster_issuer - def __init__(self, *args) -> NoReturn: - """Grafana Charm constructor.""" - super().__init__(*args) - - # Internal state initialization - self.state.set_default(pod_spec=None) - - self.port = GRAFANA_PORT - self.image = OCIImageResource(self, "image") - - # Registering regular events - self.framework.observe(self.on.start, self.configure_pod) - self.framework.observe(self.on.config_changed, self.configure_pod) - - # Registering required relation events - self.framework.observe(self.on.prometheus_relation_changed, self.configure_pod) - - # Registering required relation broken events - self.framework.observe(self.on.prometheus_relation_broken, self.configure_pod) - - @property - def relations_requirements(self): - return [RelationDefinition("prometheus", ["host", "port"], Unit)] - - def get_relation_state(self): - relation_state = {} - for relation_requirements in self.relations_requirements: - data = get_relation_data(self, relation_requirements) - relation_state = {**relation_state, **data} - check_missing_relation_data(relation_state, self.relations_requirements) - return relation_state - - def configure_pod(self, _=None) -> NoReturn: - """Assemble the pod spec and apply it, if possible. - - Args: - event (EventBase): Hook or Relation event that started the - function. - """ - if not self.unit.is_leader(): - self.unit.status = ActiveStatus("ready") - return - - relation_state = None - try: - relation_state = self.get_relation_state() - except RelationsMissing as exc: - logger.exception("Relation missing error") - self.unit.status = BlockedStatus(exc.message) - return - - self.unit.status = MaintenanceStatus("Assembling pod spec") - - # Fetch image information - try: - self.unit.status = MaintenanceStatus("Fetching image information") - image_info = self.image.fetch() - except OCIImageResourceError: - self.unit.status = BlockedStatus("Error fetching image information") - return - - try: - pod_spec = make_pod_spec( - image_info, - self.model.config, - relation_state, - self.model.app.name, - self.port, + if parsed.scheme == "https": + ingress_resource_builder.add_tls( + [parsed.hostname], config.tls_secret_name + ) + else: + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" + + ingress_resource_builder.add_rule( + parsed.hostname, self.app.name, config.port ) - except ValueError as exc: - logger.exception("Config/Relation data validation error") - self.unit.status = BlockedStatus(str(exc)) - return + ingress_resource = ingress_resource_builder.build() + pod_spec_builder.add_ingress_resource(ingress_resource) + return pod_spec_builder.build() - if self.state.pod_spec != pod_spec: - self.model.pod.set_spec(pod_spec) - self.state.pod_spec = pod_spec - self.unit.status = ActiveStatus("ready") +def _generate_random_password(): + return secrets.token_hex(16) if __name__ == "__main__":