X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=jenkins%2Fci-pipelines%2Fci_stage_3.groovy;h=90cd9f7b0166a4f64185bc98cac3aad59f05ce3b;hb=8375da1695c98bd23fcc226f0e26c20ac5710d1b;hp=620faba17fa9d52c2d8f21cece9913eb8aed9385;hpb=4a9860409030d08ad8371f89885d66a8bab66f09;p=osm%2Fdevops.git diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 620faba1..90cd9f7b 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -31,7 +31,10 @@ properties([ string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), string(defaultValue: 'osm-stage_4', description: '', name: 'DOWNSTREAM_STAGE_NAME'), string(defaultValue: 'testing-daily', description: '', name: 'DOCKER_TAG'), - string(defaultValue: 'ubuntu20.04', description: '', name: 'OPENSTACK_BASE_IMAGE'), + string(defaultValue: 'ubuntu22.04', description: '', name: 'OPENSTACK_BASE_IMAGE'), + string(defaultValue: 'osm.sanity', description: '', name: 'OPENSTACK_OSM_FLAVOR'), + booleanParam(defaultValue: false, description: '', name: 'TRY_OLD_SERVICE_ASSURANCE'), + booleanParam(defaultValue: true, description: '', name: 'TRY_JUJU_INSTALLATION'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_FAIL'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'), booleanParam(defaultValue: true, description: '', name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS'), @@ -124,13 +127,13 @@ void run_robot_systest(String tagName, void archive_logs(Map remote) { - sshCommand remote: remote, command: '''mkdir -p logs''' + sshCommand remote: remote, command: '''mkdir -p logs/dags''' if (useCharmedInstaller) { sshCommand remote: remote, command: ''' - for container in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do - logfile=`echo $container | cut -d- -f1` + for pod in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do + logfile=`echo $pod | cut -d- -f1` echo "Extracting log for $logfile" - kubectl logs -n osm $container --timestamps=true 2>&1 > logs/$logfile.log + kubectl logs -n osm $pod --timestamps=true 2>&1 > logs/$logfile.log done ''' } else { @@ -148,13 +151,17 @@ void archive_logs(Map remote) { > logs/$statefulset.log done ''' + sshCommand remote: remote, command: ''' + schedulerPod="$(kubectl get pods -n osm | grep airflow-scheduler| awk '{print $1; exit}')"; \ + echo "Extracting logs from Airflow DAGs from pod ${schedulerPod}"; \ + kubectl cp -n osm ${schedulerPod}:/opt/airflow/logs/scheduler/latest/dags logs/dags -c scheduler + ''' } sh 'rm -rf logs' sshCommand remote: remote, command: '''ls -al logs''' sshGet remote: remote, from: 'logs', into: '.', override: true - sh 'cp logs/* .' - archiveArtifacts artifacts: '*.log' + archiveArtifacts artifacts: 'logs/*.log, logs/dags/*.log' } String get_value(String key, String output) { @@ -395,7 +402,7 @@ node("${params.NODE}") { println('Launching new VM') output = sh(returnStdout: true, script: """#!/bin/sh -e for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done - openstack server create --flavor osm.sanity \ + openstack server create --flavor ${OPENSTACK_OSM_FLAVOR} \ --image ${OPENSTACK_BASE_IMAGE} \ --key-name CICD \ --property build_url="${BUILD_URL}" \ @@ -437,6 +444,30 @@ node("${params.NODE}") { } } println('VM is ready and accepting ssh connections') + + ////////////////////////////////////////////////////////////////////////////////////////////// + println('Applying sshd config workaround for Ubuntu 22.04 and old jsch client in Jenkins...') + + sh( returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "ubuntu@${IP_ADDRESS} " + + "'echo HostKeyAlgorithms +ssh-rsa | sudo tee -a /etc/ssh/sshd_config'") + sh( returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "ubuntu@${IP_ADDRESS} " + + "'echo PubkeyAcceptedKeyTypes +ssh-rsa | sudo tee -a /etc/ssh/sshd_config'") + sh( returnStatus: true, + script: "ssh -T -i ${SSH_KEY} " + + "-o StrictHostKeyChecking=no " + + "-o UserKnownHostsFile=/dev/null " + + "ubuntu@${IP_ADDRESS} " + + "'sudo systemctl restart sshd'") + ////////////////////////////////////////////////////////////////////////////////////////////// + } // stage("Spawn Remote VM") /////////////////////////////////////////////////////////////////////////////////////// @@ -456,8 +487,8 @@ node("${params.NODE}") { // Ensure the VM is ready sshCommand remote: remote, command: 'cloud-init status --wait' // Force time sync to avoid clock drift and invalid certificates - sshCommand remote: remote, command: 'sudo apt-get update' - sshCommand remote: remote, command: 'sudo apt-get install -y chrony' + sshCommand remote: remote, command: 'sudo apt-get -y update' + sshCommand remote: remote, command: 'sudo apt-get -y install chrony' sshCommand remote: remote, command: 'sudo service chrony stop' sshCommand remote: remote, command: 'sudo chronyd -vq' sshCommand remote: remote, command: 'sudo service chrony start' @@ -501,7 +532,7 @@ node("${params.NODE}") { ] sshCommand remote: remote, command: ''' - wget https://osm-download.etsi.org/ftp/osm-13.0-thirteen/install_osm.sh + wget https://osm-download.etsi.org/ftp/osm-14.0-fourteen/install_osm.sh chmod +x ./install_osm.sh sed -i '1 i\\export PATH=/snap/bin:\$PATH' ~/.bashrc ''' @@ -536,6 +567,13 @@ node("${params.NODE}") { osmHostname = "nbi.${IP_ADDRESS}.nip.io:443" } else { // Run -k8s installer here specifying internal docker registry and docker proxy + osm_installation_options = "" + if (params.TRY_OLD_SERVICE_ASSURANCE) { + osm_installation_options = "${osm_installation_options} --old-sa" + } + if (params.TRY_JUJU_INSTALLATION) { + osm_installation_options = "${osm_installation_options} --juju --lxd" + } withCredentials([gitlabCredentialsMap]) { sshCommand remote: remote, command: """ ./install_osm.sh -y \ @@ -544,7 +582,8 @@ node("${params.NODE}") { ${release} -r unstable \ -d ${USERNAME}:${PASSWORD}@${INTERNAL_DOCKER_REGISTRY} \ -p ${INTERNAL_DOCKER_PROXY} \ - -t ${containerName} + -t ${containerName} \ + ${osm_installation_options} """ } prometheusHostname = IP_ADDRESS @@ -638,7 +677,7 @@ EOF""" parallelSteps[module] = { dir("$module") { sh("docker pull ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}") - sh("""docker tag opensourcemano/${moduleName}:${moduleTag} \ + sh("""docker tag ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag} \ opensourcemano/${moduleName}:${dockerTag}""") sh "docker push opensourcemano/${moduleName}:${dockerTag}" } @@ -677,23 +716,25 @@ EOF""" } // stage('Snap promotion') stage('Charm promotion') { charms = [ - 'osm', // bundle - 'osm-ha', // bundle - 'osm-grafana', - 'osm-mariadb', - 'mongodb-exporter-k8s', - 'mysqld-exporter-k8s', - 'osm-lcm', - 'osm-mon', - 'osm-nbi', - 'osm-ng-ui', - 'osm-pol', - 'osm-ro', - 'osm-prometheus', - 'osm-vca-integrator', + [name: 'osm', base: "22.04"], + [name: 'osm-ha', base: "22.04"], + [name: 'mongodb-exporter-k8s', base: "20.04"], + [name: 'mysqld-exporter-k8s', base: "20.04"], + [name: 'osm-grafana', base: "20.04"], + [name: 'osm-keystone', base: "22.04"], + [name: 'osm-lcm', base: "22.04"], + [name: 'osm-mon', base: "22.04"], + [name: 'osm-nbi', base: "22.04"], + [name: 'osm-ng-ui', base: "22.04"], + [name: 'osm-pol', base: "22.04"], + [name: 'osm-ro', base: "22.04"], + [name: 'osm-prometheus', base: "20.04"], + [name: 'osm-update-db-operator', base: "20.04"], + [name: 'osm-vca-integrator', base: "22.04"], ] - for (charm in charms) { - + for (entry in charms) { + charm = entry.name + base = entry.base channel = 'latest' if (BRANCH_NAME.startsWith('v')) { channel = BRANCH_NAME.substring(1) @@ -703,15 +744,15 @@ EOF""" withCredentials([string(credentialsId: 'Charmstore', variable: 'CHARMCRAFT_AUTH')]) { sh "charmcraft status $charm --format json > ${charm}.json" - isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l").trim() as int + isCharm = sh(returnStdout: true, script: "grep architecture ${charm}.json | wc -l 2>/dev/null").trim() as int resourceArgument = "" if (isCharm) { - jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .version'|head -1" - jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"$channel/beta\")| .version'|head -1" + jqScriptEdge = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"$base\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .version'|head -1" + jqScriptBeta = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"$base\") | .releases[] | select(.channel==\"$channel/beta\")| .version'|head -1" index=0 while (index < 5) { - resourceNameScript = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .resources[$index].name'|head -1" - resourceRevsScript = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"20.04\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .resources[$index].revision'|head -1" + resourceNameScript = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"$base\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .resources[$index].name'|head -1" + resourceRevsScript = "cat ${charm}.json | jq -r '.[] | select(.track==\"$channel\") | .mappings[] | select(.base.architecture==\"amd64\" and .base.channel==\"$base\") | .releases[] | select(.channel==\"$channel/edge/merged\")| .resources[$index].revision'|head -1" resourceName = sh(returnStdout: true, script: resourceNameScript).trim() resourceRevs = sh(returnStdout: true, script: resourceRevsScript).trim() if (resourceName != "null") { @@ -747,7 +788,10 @@ EOF""" } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) } // dir(OSM_DEVOPS) } finally { - stage('Archive Container Logs') { + // stage('Debug') { + // sleep 900 + // } + stage('Archive Container Logs') { if ( ARCHIVE_LOGS_FLAG ) { try { // Archive logs