Add more logs for troubleshoot problems in management cluster

Change-Id: Iedc18221ce3150db83e8ed4438e6ee4ba4bbc5eb
Signed-off-by: mesaj <juanmanuel.mesamendez.ext@telefonica.com>

diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy
index af0f770..eaf6009 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -516,14 +516,41 @@ void run_robot_systest(String tagName,
 
 void archive_logs(Map remote) {
 
-    sshCommand remote: remote, command: '''mkdir -p logs/dags logs/vcluster logs/events'''
+    sshCommand remote: remote, command: '''mkdir -p logs/dags logs/vcluster logs/flux-system logs/events logs/system'''
     // Collect Kubernetes events
     sshCommand remote: remote, command: '''
         echo "Extracting Kubernetes events"
         kubectl get events --all-namespaces --sort-by='.lastTimestamp' -o wide > logs/events/k8s-events.log 2>&1 || true
         kubectl get events -n osm --sort-by='.lastTimestamp' -o wide > logs/events/osm-events.log 2>&1 || true
         kubectl get events -n vcluster --sort-by='.lastTimestamp' -o wide > logs/events/vcluster-events.log 2>&1 || true
+        kubectl get events -n flux-system --sort-by='.lastTimestamp' -o wide > logs/events/flux-system-events.log 2>&1 || true
     '''
+    // Collect host logs
+    sshCommand remote: remote, command: '''
+      echo "Collect system logs"
+      if command -v journalctl >/dev/null; then
+        journalctl > logs/system/system.log
+      fi
+
+      for entry in syslog messages; do
+        [ -e "/var/log/${entry}" ] && cp -f /var/log/${entry} logs/system/"${entry}.log"
+      done
+
+      echo "Collect active services"
+      case "$(cat /proc/1/comm)" in
+        systemd)
+          systemctl list-units > logs/system/services.txt 2>&1
+          ;;
+        *)
+          service --status-all >> logs/system/services.txt 2>&1
+          ;;
+      esac
+
+      top -b -n 1 > logs/system/top.txt 2>&1
+      ps fauxwww > logs/system/ps.txt 2>&1
+    '''
+
+
     if (useCharmedInstaller) {
         sshCommand remote: remote, command: '''
             for pod in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do
@@ -561,12 +588,21 @@ void archive_logs(Map remote) {
                 > logs/vcluster/$pod.log || true
             done
         '''
+        // Collect flux-system namespace logs
+        sshCommand remote: remote, command: '''
+            echo "Extracting logs from flux-system namespace"
+            for pod in `kubectl get pods -n flux-system | grep -v NAME | awk '{print $1}'`; do
+                echo "Extracting log for flux-system pod: $pod"
+                kubectl logs -n flux-system $pod --timestamps=true --all-containers 2>&1 \
+                > logs/flux-system/$pod.log || true
+            done
+        '''
     }
 
     sh 'rm -rf logs'
-    sshCommand remote: remote, command: '''ls -al logs logs/vcluster logs/events'''
+    sshCommand remote: remote, command: '''ls -al logs logs/vcluster logs/events logs/flux-system logs/system'''
     sshGet remote: remote, from: 'logs', into: '.', override: true
-    archiveArtifacts artifacts: 'logs/*.log, logs/dags/*.log, logs/vcluster/*.log, logs/events/*.log'
+    archiveArtifacts artifacts: 'logs/*.log, logs/dags/*.log, logs/vcluster/*.log, logs/events/*.log, logs/flux-system/*.log, logs/system/**'
 }
 
 String get_value(String key, String output) {