From: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Fri, 4 Oct 2024 11:13:59 +0000 (+0200)
Subject: Feature 11050: performance optimization in Airflow
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F27%2F14627%2F5;p=osm%2Fdevops.git

Feature 11050: performance optimization in Airflow

Change-Id: Ieb5aa36723d3a06850c3839e12916477efb2c4f5
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
---

diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml
index bcfe583c..798648c5 100644
--- a/installers/helm/values/airflow-values.yaml
+++ b/installers/helm/values/airflow-values.yaml
@@ -12,38 +12,75 @@
 #   limitations under the License.
 #
 ---
+config:
+  core:
+    parallelism: 4
+    dag_concurrency: 2
+    max_active_tasks_per_dag: 2
+    worker_concurrency: 2
+    dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa"
 defaultAirflowRepository: opensourcemano/airflow
 defaultAirflowTag: "testing-daily"
+executor: LocalExecutor
+extraEnv: |
+  - name: AIRFLOW__API__AUTH_BACKENDS
+    value: 'airflow.api.auth.backend.basic_auth'
+  - name: AIRFLOW__LOGGING__LOGGING_LEVEL
+    value: 'INFO'
+  - name: OSMMON_DATABASE_COMMONKEY
+    valueFrom:
+      secretKeyRef:
+        name: osm-secret
+        key: OSM_DATABASE_COMMONKEY
+# extraEnvFrom: |
+#   - secretRef:
+#       name: ngsa-secret
 ingress:
   web:
     enabled: True
     hosts:
       - name: "localhost"
     ingressClassName: nginx
+# scheduler:
+#   resources:
+#     requests:
+#       memory: 512Mi
+#       cpu: 250m
+#     limits:
+#       memory: 1Gi
+#       cpu: 500m
 webserverSecretKeySecretName: airflow-webserver-secret
 webserver:
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 250m
+    limits:
+      memory: 1Gi
+      cpu: 500m
   service:
     type: ClusterIP
     ports:
       - name: airflow-ui
         port: "{{ .Values.ports.airflowUI }}"
         targetPort: "{{ .Values.ports.airflowUI }}"
-extraEnv: |
-  - name: AIRFLOW__API__AUTH_BACKENDS
-    value: 'airflow.api.auth.backend.basic_auth'
-  - name: AIRFLOW__LOGGING__LOGGING_LEVEL
-    value: 'INFO'
-  - name: OSMMON_DATABASE_COMMONKEY
-    valueFrom:
-      secretKeyRef:
-        name: osm-secret
-        key: OSM_DATABASE_COMMONKEY
-# extraEnvFrom: |
-#   - secretRef:
-#       name: ngsa-secret
-config:
-  core:
-    dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa"
 workers:
   persistence:
-    size: 8Gi
+    size: 2Gi
+  replicas: 1
+  resources:
+    requests:
+      memory: 512Mi
+      cpu: 250m
+    limits:
+      memory: 1Gi
+      cpu: 500m
+  logGroomerSidecar:
+    retentionDays: 7
+    resources:
+      limits:
+        cpu: 100m
+        memory: 128Mi
+      requests:
+        cpu: 100m
+        memory: 128Mi