From: garciadeblas Date: Fri, 4 Oct 2024 11:13:59 +0000 (+0200) Subject: Feature 11050: performance optimization in Airflow X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=8ef3b053a941555a4b6173f2c64ec722374600e3;p=osm%2Fdevops.git Feature 11050: performance optimization in Airflow Change-Id: Ieb5aa36723d3a06850c3839e12916477efb2c4f5 Signed-off-by: garciadeblas --- diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml index bcfe583c..798648c5 100644 --- a/installers/helm/values/airflow-values.yaml +++ b/installers/helm/values/airflow-values.yaml @@ -12,38 +12,75 @@ # limitations under the License. # --- +config: + core: + parallelism: 4 + dag_concurrency: 2 + max_active_tasks_per_dag: 2 + worker_concurrency: 2 + dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa" defaultAirflowRepository: opensourcemano/airflow defaultAirflowTag: "testing-daily" +executor: LocalExecutor +extraEnv: | + - name: AIRFLOW__API__AUTH_BACKENDS + value: 'airflow.api.auth.backend.basic_auth' + - name: AIRFLOW__LOGGING__LOGGING_LEVEL + value: 'INFO' + - name: OSMMON_DATABASE_COMMONKEY + valueFrom: + secretKeyRef: + name: osm-secret + key: OSM_DATABASE_COMMONKEY +# extraEnvFrom: | +# - secretRef: +# name: ngsa-secret ingress: web: enabled: True hosts: - name: "localhost" ingressClassName: nginx +# scheduler: +# resources: +# requests: +# memory: 512Mi +# cpu: 250m +# limits: +# memory: 1Gi +# cpu: 500m webserverSecretKeySecretName: airflow-webserver-secret webserver: + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m service: type: ClusterIP ports: - name: airflow-ui port: "{{ .Values.ports.airflowUI }}" targetPort: "{{ .Values.ports.airflowUI }}" -extraEnv: | - - name: AIRFLOW__API__AUTH_BACKENDS - value: 'airflow.api.auth.backend.basic_auth' - - name: AIRFLOW__LOGGING__LOGGING_LEVEL - value: 'INFO' - - name: OSMMON_DATABASE_COMMONKEY - valueFrom: - secretKeyRef: - name: osm-secret - key: OSM_DATABASE_COMMONKEY -# extraEnvFrom: | -# - secretRef: -# name: ngsa-secret -config: - core: - dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa" workers: persistence: - size: 8Gi + size: 2Gi + replicas: 1 + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m + logGroomerSidecar: + retentionDays: 7 + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi