X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=installers%2Fcharm%2Fnbi%2Fsrc%2Fcharm.py;h=1460459a813e9959a79317af0d7cc3a305ca0dde;hb=refs%2Fchanges%2F27%2F11127%2F2;hp=3970a33d6a639d191425daa7e6755e0dca3afb73;hpb=4df5a461573ae47200e893453c7c7d9cb89da30d;p=osm%2Fdevops.git diff --git a/installers/charm/nbi/src/charm.py b/installers/charm/nbi/src/charm.py index 3970a33d..1460459a 100755 --- a/installers/charm/nbi/src/charm.py +++ b/installers/charm/nbi/src/charm.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 Canonical Ltd. +# Copyright 2021 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,370 +20,308 @@ # osm-charmers@lists.launchpad.net ## -import logging -from typing import Any, Dict, NoReturn -from pydantic import ValidationError - -from ops.charm import CharmBase, CharmEvents -from ops.framework import EventBase, EventSource, StoredState -from ops.main import main -from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus -from oci_image import OCIImageResource, OCIImageResourceError - -from pod_spec import make_pod_spec - -logger = logging.getLogger(__name__) - -NBI_PORT = 9999 +# pylint: disable=E0213 -class ConfigurePodEvent(EventBase): - """Configure Pod event""" - - pass +from ipaddress import ip_network +import logging +from typing import NoReturn, Optional +from urllib.parse import urlparse -class NbiEvents(CharmEvents): - """NBI Events""" +from ops.main import main +from opslib.osm.charm import CharmedOsmBase, RelationsMissing +from opslib.osm.interfaces.http import HttpServer +from opslib.osm.interfaces.kafka import KafkaClient +from opslib.osm.interfaces.keystone import KeystoneClient +from opslib.osm.interfaces.mongo import MongoClient +from opslib.osm.interfaces.prometheus import PrometheusClient +from opslib.osm.pod import ( + ContainerV3Builder, + IngressResourceV3Builder, + PodSpecV3Builder, +) +from opslib.osm.validator import ModelValidator, validator - configure_pod = EventSource(ConfigurePodEvent) +logger = logging.getLogger(__name__) -class NbiCharm(CharmBase): - """NBI Charm.""" +PORT = 9999 + + +class ConfigModel(ModelValidator): + enable_test: bool + auth_backend: str + database_commonkey: str + log_level: str + max_file_size: int + site_url: Optional[str] + cluster_issuer: Optional[str] + ingress_class: Optional[str] + ingress_whitelist_source_range: Optional[str] + tls_secret_name: Optional[str] + mongodb_uri: Optional[str] + image_pull_policy: str + + @validator("auth_backend") + def validate_auth_backend(cls, v): + if v not in {"internal", "keystone"}: + raise ValueError("value must be 'internal' or 'keystone'") + return v + + @validator("log_level") + def validate_log_level(cls, v): + if v not in {"INFO", "DEBUG"}: + raise ValueError("value must be INFO or DEBUG") + return v + + @validator("max_file_size") + def validate_max_file_size(cls, v): + if v < 0: + raise ValueError("value must be equal or greater than 0") + return v + + @validator("site_url") + def validate_site_url(cls, v): + if v: + parsed = urlparse(v) + if not parsed.scheme.startswith("http"): + raise ValueError("value must start with http") + return v + + @validator("ingress_whitelist_source_range") + def validate_ingress_whitelist_source_range(cls, v): + if v: + ip_network(v) + return v + + @validator("mongodb_uri") + def validate_mongodb_uri(cls, v): + if v and not v.startswith("mongodb://"): + raise ValueError("mongodb_uri is not properly formed") + return v + + @validator("image_pull_policy") + def validate_image_pull_policy(cls, v): + values = { + "always": "Always", + "ifnotpresent": "IfNotPresent", + "never": "Never", + } + v = v.lower() + if v not in values.keys(): + raise ValueError("value must be always, ifnotpresent or never") + return values[v] - state = StoredState() - on = NbiEvents() +class NbiCharm(CharmedOsmBase): def __init__(self, *args) -> NoReturn: - """NBI Charm constructor.""" - super().__init__(*args) - - # Internal state initialization - self.state.set_default(pod_spec=None) - - # Message bus data initialization - self.state.set_default(message_host=None) - self.state.set_default(message_port=None) - - # Database data initialization - self.state.set_default(database_uri=None) - - # Prometheus data initialization - self.state.set_default(prometheus_host=None) - self.state.set_default(prometheus_port=None) - - # Keystone data initialization - self.state.set_default(keystone_host=None) - self.state.set_default(keystone_port=None) - self.state.set_default(keystone_user_domain_name=None) - self.state.set_default(keystone_project_domain_name=None) - self.state.set_default(keystone_username=None) - self.state.set_default(keystone_password=None) - self.state.set_default(keystone_service=None) - - self.port = NBI_PORT - self.image = OCIImageResource(self, "image") - - # Registering regular events - self.framework.observe(self.on.start, self.configure_pod) - self.framework.observe(self.on.config_changed, self.configure_pod) - self.framework.observe(self.on.upgrade_charm, self.configure_pod) + super().__init__( + *args, + oci_image="image", + debug_mode_config_key="debug_mode", + debug_pubkey_config_key="debug_pubkey", + vscode_workspace=VSCODE_WORKSPACE, + ) - # Registering custom internal events - self.framework.observe(self.on.configure_pod, self.configure_pod) + self.kafka_client = KafkaClient(self, "kafka") + self.framework.observe(self.on["kafka"].relation_changed, self.configure_pod) + self.framework.observe(self.on["kafka"].relation_broken, self.configure_pod) - # Registering required relation changed events - self.framework.observe( - self.on.kafka_relation_changed, self._on_kafka_relation_changed - ) - self.framework.observe( - self.on.mongodb_relation_changed, self._on_mongodb_relation_changed - ) - self.framework.observe( - self.on.keystone_relation_changed, self._on_keystone_relation_changed - ) - self.framework.observe( - self.on.prometheus_relation_changed, self._on_prometheus_relation_changed - ) + self.mongodb_client = MongoClient(self, "mongodb") + self.framework.observe(self.on["mongodb"].relation_changed, self.configure_pod) + self.framework.observe(self.on["mongodb"].relation_broken, self.configure_pod) - # Registering required relation departed events - self.framework.observe( - self.on.kafka_relation_departed, self._on_kafka_relation_departed - ) - self.framework.observe( - self.on.mongodb_relation_departed, self._on_mongodb_relation_departed - ) + self.prometheus_client = PrometheusClient(self, "prometheus") self.framework.observe( - self.on.keystone_relation_departed, self._on_keystone_relation_departed + self.on["prometheus"].relation_changed, self.configure_pod ) self.framework.observe( - self.on.prometheus_relation_departed, self._on_prometheus_relation_departed + self.on["prometheus"].relation_broken, self.configure_pod ) - # Registering provided relation events - self.framework.observe(self.on.nbi_relation_joined, self._publish_nbi_info) + self.keystone_client = KeystoneClient(self, "keystone") + self.framework.observe(self.on["keystone"].relation_changed, self.configure_pod) + self.framework.observe(self.on["keystone"].relation_broken, self.configure_pod) - def _on_kafka_relation_changed(self, event: EventBase) -> NoReturn: - """Reads information about the kafka relation. - - Args: - event (EventBase): Kafka relation event. - """ - data_loc = event.unit if event.unit else event.app - - message_host = event.relation.data[data_loc].get("host") - message_port = event.relation.data[data_loc].get("port") - - if ( - message_host - and message_port - and ( - self.state.message_host != message_host - or self.state.message_port != message_port - ) - ): - self.state.message_host = message_host - self.state.message_port = message_port - self.on.configure_pod.emit() + self.http_server = HttpServer(self, "nbi") + self.framework.observe(self.on["nbi"].relation_joined, self._publish_nbi_info) - def _on_kafka_relation_departed(self, event: EventBase) -> NoReturn: - """Clears data from kafka relation. - - Args: - event (EventBase): Kafka relation event. - """ - self.state.message_host = None - self.state.message_port = None - self.on.configure_pod.emit() - - def _on_mongodb_relation_changed(self, event: EventBase) -> NoReturn: - """Reads information about the DB relation. - - Args: - event (EventBase): DB relation event. - """ - data_loc = event.unit if event.unit else event.app - - database_uri = event.relation.data[data_loc].get("connection_string") - - if database_uri and self.state.database_uri != database_uri: - self.state.database_uri = database_uri - self.on.configure_pod.emit() - - def _on_mongodb_relation_departed(self, event: EventBase) -> NoReturn: - """Clears data from mongodb relation. - - Args: - event (EventBase): DB relation event. - """ - self.state.database_uri = None - self.on.configure_pod.emit() - - def _on_keystone_relation_changed(self, event: EventBase) -> NoReturn: - """Reads information about the keystone relation. - - Args: - event (EventBase): Keystone relation event. - """ - data_loc = event.unit if event.unit else event.app - - keystone_host = event.relation.data[data_loc].get("host") - keystone_port = event.relation.data[data_loc].get("port") - keystone_user_domain_name = event.relation.data[data_loc].get( - "user_domain_name" - ) - keystone_project_domain_name = event.relation.data[data_loc].get( - "project_domain_name" - ) - keystone_username = event.relation.data[data_loc].get("username") - keystone_password = event.relation.data[data_loc].get("password") - keystone_service = event.relation.data[data_loc].get("service") - - if ( - keystone_host - and keystone_port - and keystone_user_domain_name - and keystone_project_domain_name - and keystone_username - and keystone_password - and keystone_service - and ( - self.state.keystone_host != keystone_host - or self.state.keystone_port != keystone_port - or self.state.keystone_user_domain_name != keystone_user_domain_name - or self.state.keystone_project_domain_name - != keystone_project_domain_name - or self.state.keystone_username != keystone_username - or self.state.keystone_password != keystone_password - or self.state.keystone_service != keystone_service - ) - ): - self.state.keystone_host = keystone_host - self.state.keystone_port = keystone_port - self.state.keystone_user_domain_name = keystone_user_domain_name - self.state.keystone_project_domain_name = keystone_project_domain_name - self.state.keystone_username = keystone_username - self.state.keystone_password = keystone_password - self.state.keystone_service = keystone_service - self.on.configure_pod.emit() - - def _on_keystone_relation_departed(self, event: EventBase) -> NoReturn: - """Clears data from keystone relation. - - Args: - event (EventBase): Keystone relation event. - """ - self.state.keystone_host = None - self.state.keystone_port = None - self.state.keystone_user_domain_name = None - self.state.keystone_project_domain_name = None - self.state.keystone_username = None - self.state.keystone_password = None - self.state.keystone_service = None - self.on.configure_pod.emit() - - def _on_prometheus_relation_changed(self, event: EventBase) -> NoReturn: - """Reads information about the prometheus relation. - - Args: - event (EventBase): Prometheus relation event. - """ - data_loc = event.unit if event.unit else event.app - - prometheus_host = event.relation.data[data_loc].get("hostname") - prometheus_port = event.relation.data[data_loc].get("port") - - if ( - prometheus_host - and prometheus_port - and ( - self.state.prometheus_host != prometheus_host - or self.state.prometheus_port != prometheus_port - ) - ): - self.state.prometheus_host = prometheus_host - self.state.prometheus_port = prometheus_port - self.on.configure_pod.emit() - - def _on_prometheus_relation_departed(self, event: EventBase) -> NoReturn: - """Clears data from prometheus relation. - - Args: - event (EventBase): Prometheus relation event. - """ - self.state.prometheus_host = None - self.state.prometheus_port = None - self.on.configure_pod.emit() - - def _publish_nbi_info(self, event: EventBase) -> NoReturn: + def _publish_nbi_info(self, event): """Publishes NBI information. Args: - event (EventBase): NBI relation event. + event (EventBase): RO relation event. """ if self.unit.is_leader(): - rel_data = { - "host": self.model.app.name, - "port": str(NBI_PORT), + self.http_server.publish_info(self.app.name, PORT) + + def _check_missing_dependencies(self, config: ConfigModel): + missing_relations = [] + + if self.kafka_client.is_missing_data_in_unit(): + missing_relations.append("kafka") + if not config.mongodb_uri and self.mongodb_client.is_missing_data_in_unit(): + missing_relations.append("mongodb") + if self.prometheus_client.is_missing_data_in_app(): + missing_relations.append("prometheus") + if config.auth_backend == "keystone": + if self.keystone_client.is_missing_data_in_app(): + missing_relations.append("keystone") + + if missing_relations: + raise RelationsMissing(missing_relations) + + def build_pod_spec(self, image_info): + # Validate config + config = ConfigModel(**dict(self.config)) + + if config.mongodb_uri and not self.mongodb_client.is_missing_data_in_unit(): + raise Exception("Mongodb data cannot be provided via config and relation") + + # Check relations + self._check_missing_dependencies(config) + + # Create Builder for the PodSpec + pod_spec_builder = PodSpecV3Builder() + + # Build Init Container + pod_spec_builder.add_init_container( + { + "name": "init-check", + "image": "alpine:latest", + "command": [ + "sh", + "-c", + f"until (nc -zvw1 {self.kafka_client.host} {self.kafka_client.port} ); do sleep 3; done; exit 0", + ], } - for k, v in rel_data.items(): - event.relation.data[self.model.app][k] = v - - def _missing_relations(self) -> str: - """Checks if there missing relations. - - Returns: - str: string with missing relations - """ - data_status = { - "kafka": self.state.message_host, - "mongodb": self.state.database_uri, - "prometheus": self.state.prometheus_host, - } - - if self.model.config["auth_backend"] == "keystone": - data_status["keystone"] = self.state.keystone_host - - missing_relations = [k for k, v in data_status.items() if not v] - - return ", ".join(missing_relations) - - @property - def relation_state(self) -> Dict[str, Any]: - """Collects relation state configuration for pod spec assembly. - - Returns: - Dict[str, Any]: relation state information. - """ - relation_state = { - "message_host": self.state.message_host, - "message_port": self.state.message_port, - "database_uri": self.state.database_uri, - "prometheus_host": self.state.prometheus_host, - "prometheus_port": self.state.prometheus_port, - } + ) - if self.model.config["auth_backend"] == "keystone": - relation_state.update( + # Build Container + container_builder = ContainerV3Builder( + self.app.name, image_info, config.image_pull_policy + ) + container_builder.add_port(name=self.app.name, port=PORT) + container_builder.add_tcpsocket_readiness_probe( + PORT, + initial_delay_seconds=5, + timeout_seconds=5, + ) + container_builder.add_tcpsocket_liveness_probe( + PORT, + initial_delay_seconds=45, + timeout_seconds=10, + ) + container_builder.add_envs( + { + # General configuration + "ALLOW_ANONYMOUS_LOGIN": "yes", + "OSMNBI_SERVER_ENABLE_TEST": config.enable_test, + "OSMNBI_STATIC_DIR": "/app/osm_nbi/html_public", + # Kafka configuration + "OSMNBI_MESSAGE_HOST": self.kafka_client.host, + "OSMNBI_MESSAGE_DRIVER": "kafka", + "OSMNBI_MESSAGE_PORT": self.kafka_client.port, + # Database configuration + "OSMNBI_DATABASE_DRIVER": "mongo", + "OSMNBI_DATABASE_URI": config.mongodb_uri + or self.mongodb_client.connection_string, + "OSMNBI_DATABASE_COMMONKEY": config.database_commonkey, + # Storage configuration + "OSMNBI_STORAGE_DRIVER": "mongo", + "OSMNBI_STORAGE_PATH": "/app/storage", + "OSMNBI_STORAGE_COLLECTION": "files", + "OSMNBI_STORAGE_URI": config.mongodb_uri + or self.mongodb_client.connection_string, + # Prometheus configuration + "OSMNBI_PROMETHEUS_HOST": self.prometheus_client.hostname, + "OSMNBI_PROMETHEUS_PORT": self.prometheus_client.port, + # Log configuration + "OSMNBI_LOG_LEVEL": config.log_level, + } + ) + if config.auth_backend == "internal": + container_builder.add_env("OSMNBI_AUTHENTICATION_BACKEND", "internal") + elif config.auth_backend == "keystone": + container_builder.add_envs( { - "keystone_host": self.state.keystone_host, - "keystone_port": self.state.keystone_port, - "keystone_user_domain_name": self.state.keystone_user_domain_name, - "keystone_project_domain_name": self.state.keystone_project_domain_name, - "keystone_username": self.state.keystone_username, - "keystone_password": self.state.keystone_password, - "keystone_service": self.state.keystone_service, + "OSMNBI_AUTHENTICATION_BACKEND": "keystone", + "OSMNBI_AUTHENTICATION_AUTH_URL": self.keystone_client.host, + "OSMNBI_AUTHENTICATION_AUTH_PORT": self.keystone_client.port, + "OSMNBI_AUTHENTICATION_USER_DOMAIN_NAME": self.keystone_client.user_domain_name, + "OSMNBI_AUTHENTICATION_PROJECT_DOMAIN_NAME": self.keystone_client.project_domain_name, + "OSMNBI_AUTHENTICATION_SERVICE_USERNAME": self.keystone_client.username, + "OSMNBI_AUTHENTICATION_SERVICE_PASSWORD": self.keystone_client.password, + "OSMNBI_AUTHENTICATION_SERVICE_PROJECT": self.keystone_client.service, } ) - - return relation_state - - def configure_pod(self, event: EventBase) -> NoReturn: - """Assemble the pod spec and apply it, if possible. - - Args: - event (EventBase): Hook or Relation event that started the - function. - """ - if missing := self._missing_relations(): - self.unit.status = BlockedStatus( - f"Waiting for {missing} relation{'s' if ',' in missing else ''}" - ) - return - - if not self.unit.is_leader(): - self.unit.status = ActiveStatus("ready") - return - - self.unit.status = MaintenanceStatus("Assembling pod spec") - - # Fetch image information - try: - self.unit.status = MaintenanceStatus("Fetching image information") - image_info = self.image.fetch() - except OCIImageResourceError: - self.unit.status = BlockedStatus("Error fetching image information") - return - - try: - pod_spec = make_pod_spec( - image_info, - self.model.config, - self.relation_state, - self.model.app.name, - self.port, + container = container_builder.build() + + # Add container to pod spec + pod_spec_builder.add_container(container) + + # Add ingress resources to pod spec if site url exists + if config.site_url: + parsed = urlparse(config.site_url) + annotations = { + "nginx.ingress.kubernetes.io/proxy-body-size": "{}".format( + str(config.max_file_size) + "m" + if config.max_file_size > 0 + else config.max_file_size + ), + "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", + } + if config.ingress_class: + annotations["kubernetes.io/ingress.class"] = config.ingress_class + ingress_resource_builder = IngressResourceV3Builder( + f"{self.app.name}-ingress", annotations ) - except ValidationError as exc: - logger.exception("Config/Relation data validation error") - self.unit.status = BlockedStatus(str(exc)) - return - - if self.state.pod_spec != pod_spec: - self.model.pod.set_spec(pod_spec) - self.state.pod_spec = pod_spec - self.unit.status = ActiveStatus("ready") + if config.ingress_whitelist_source_range: + annotations[ + "nginx.ingress.kubernetes.io/whitelist-source-range" + ] = config.ingress_whitelist_source_range + + if config.cluster_issuer: + annotations["cert-manager.io/cluster-issuer"] = config.cluster_issuer + + if parsed.scheme == "https": + ingress_resource_builder.add_tls( + [parsed.hostname], config.tls_secret_name + ) + else: + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" + + ingress_resource_builder.add_rule(parsed.hostname, self.app.name, PORT) + ingress_resource = ingress_resource_builder.build() + pod_spec_builder.add_ingress_resource(ingress_resource) + + logger.debug(pod_spec_builder.build()) + + return pod_spec_builder.build() + + +VSCODE_WORKSPACE = { + "folders": [ + {"path": "/usr/lib/python3/dist-packages/osm_nbi"}, + {"path": "/usr/lib/python3/dist-packages/osm_common"}, + {"path": "/usr/lib/python3/dist-packages/osm_im"}, + ], + "settings": {}, + "launch": { + "version": "0.2.0", + "configurations": [ + { + "name": "NBI", + "type": "python", + "request": "launch", + "module": "osm_nbi.nbi", + "justMyCode": False, + } + ], + }, +} if __name__ == "__main__":