commit 7992e28b53ab1da3fcbb5a7dd7042310dc31c82e
author garciadeblas <gerardo.garciadeblas@telefonica.com> Fri Oct 04 13:13:59 2024 +0200
committer garciadeblas <gerardo.garciadeblas@telefonica.com> Wed Mar 19 16:39:07 2025 +0100
tree a355cb12d245da57629ebef1c84dab5eb706c600
parent 8692f2a47fe313782062d520b13cf44b89c9a1e3

Feature 11050: performance optimization in Airflow

Change-Id: Ieb5aa36723d3a06850c3839e12916477efb2c4f5
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>

installers/helm/values/airflow-values.yaml

diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml
index 1b67a18..082793c 100644
--- a/installers/helm/values/airflow-values.yaml
+++ b/installers/helm/values/airflow-values.yaml
@@ -12,22 +12,16 @@
 #   limitations under the License.
 #
 ---
+config:
+  core:
+    parallelism: 4
+    dag_concurrency: 2
+    max_active_tasks_per_dag: 2
+    worker_concurrency: 2
+    dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa"
 defaultAirflowRepository: opensourcemano/airflow
 defaultAirflowTag: "16"
-ingress:
-  web:
-    enabled: True
-    hosts:
-      - name: "localhost"
-    ingressClassName: nginx
-webserverSecretKeySecretName: airflow-webserver-secret
-webserver:
-  service:
-    type: ClusterIP
-    ports:
-      - name: airflow-ui
-        port: "{{ .Values.ports.airflowUI }}"
-        targetPort: "{{ .Values.ports.airflowUI }}"
+executor: LocalExecutor
 extraEnv: |
   - name: AIRFLOW__API__AUTH_BACKENDS
     value: 'airflow.api.auth.backend.basic_auth'
@@ -41,9 +35,52 @@
 # extraEnvFrom: |
 #   - secretRef:
 #       name: ngsa-secret
-config:
-  core:
-    dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa"
+ingress:
+  web:
+    enabled: True
+    hosts:
+      - name: "localhost"
+    ingressClassName: nginx
+# scheduler:
+#   resources:
+#     requests:
+#       memory: 512Mi
+#       cpu: 250m
+#     limits:
+#       memory: 1Gi
+#       cpu: 500m
+webserverSecretKeySecretName: airflow-webserver-secret
+webserver:
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 250m
+    limits:
+      memory: 1Gi
+      cpu: 500m
+  service:
+    type: ClusterIP
+    ports:
+      - name: airflow-ui
+        port: "{{ .Values.ports.airflowUI }}"
+        targetPort: "{{ .Values.ports.airflowUI }}"
 workers:
   persistence:
-    size: 8Gi
+    size: 2Gi
+  replicas: 1
+  resources:
+    requests:
+      memory: 512Mi
+      cpu: 250m
+    limits:
+      memory: 1Gi
+      cpu: 500m
+  logGroomerSidecar:
+    retentionDays: 7
+    resources:
+      limits:
+        cpu: 100m
+        memory: 128Mi
+      requests:
+        cpu: 100m
+        memory: 128Mi