Self-contained repos: new Jenkins pipeline
Change-Id: I2f03caed37dc9e46ce6e4cba5c58bd53043fe5b4
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
Signed-off-by: mesaj <juanmanuel.mesamendez.ext@telefonica.com>
diff --git a/Dockerfile.production b/Dockerfile.production
index fac57c4..703e4c0 100644
--- a/Dockerfile.production
+++ b/Dockerfile.production
@@ -57,8 +57,7 @@
ENV PATH="/app/osm_nbi/.venv/bin:$PATH"
# Install OSM dependencies with cache optimization
-RUN --mount=type=cache,target=/root/.cache/pip \
- git clone --filter=blob:none --tags https://osm.etsi.org/gerrit/osm/common.git /tmp/osm-common && \
+RUN git clone --filter=blob:none --tags https://osm.etsi.org/gerrit/osm/common.git /tmp/osm-common && \
cd /tmp/osm-common && \
git fetch origin ${COMMON_GERRIT_REFSPEC} && \
git checkout FETCH_HEAD && \
@@ -66,8 +65,7 @@
pip install --no-cache-dir -r /tmp/osm-common/requirements.txt && \
pip install /tmp/osm-common
-RUN --mount=type=cache,target=/root/.cache/pip \
- git clone --filter=blob:none --tags https://osm.etsi.org/gerrit/osm/IM.git /tmp/osm-im && \
+RUN git clone --filter=blob:none --tags https://osm.etsi.org/gerrit/osm/IM.git /tmp/osm-im && \
cd /tmp/osm-im && \
git fetch origin ${IM_GERRIT_REFSPEC} && \
git checkout FETCH_HEAD && \
@@ -76,12 +74,10 @@
pip install /tmp/osm-im
COPY requirements.txt ./
-RUN --mount=type=cache,target=/root/.cache/pip \
- pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
COPY . .
-RUN --mount=type=cache,target=/root/.cache/pip \
- pip install .
+RUN pip install .
# Clean up
RUN find /app/osm_nbi -type d -name ".tox" -exec rm -rf {} +
diff --git a/Jenkinsfile.new b/Jenkinsfile.new
index 4d02e99..06616fe 100644
--- a/Jenkinsfile.new
+++ b/Jenkinsfile.new
@@ -13,29 +13,249 @@
limitations under the License.
*/
-pipeline {
- agent none
- parameters {
- string(defaultValue: env.BRANCH_NAME, description: '', name: 'GERRIT_BRANCH')
- string(defaultValue: 'osm/NBI', description: '', name: 'GERRIT_PROJECT')
- string(defaultValue: env.GERRIT_REFSPEC, description: '', name: 'GERRIT_REFSPEC')
- string(defaultValue: env.GERRIT_PATCHSET_REVISION, description: '', name: 'GERRIT_PATCHSET_REVISION')
- string(defaultValue: 'https://osm.etsi.org/gerrit', description: '', name: 'PROJECT_URL_PREFIX')
- booleanParam(defaultValue: false, description: '', name: 'TEST_INSTALL')
- // string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER')
- // New parameters for Docker image build
- string(defaultValue: 'opensourcemano/nbi', description: 'Docker Image Name', name: 'IMAGENAME')
- string(defaultValue: 'localhost:5000', description: 'Docker Registry', name: 'DOCKER_REGISTRY')
- string(defaultValue: 'http://', description: 'Docker Registry protocol', name: 'DOCKER_REGISTRY_PROTOCOL')
- string(defaultValue: '', description: 'ID of Docker Registry Credentials', name: 'DOCKER_CREDENTIALS') // `defaultValue` to be updated with actual ID in Jenkins whenever needed
- }
- stages {
- stage('TEST') {
- agent { label 'system' }
- steps {
- echo "HELLO"
- }
- }
- }
-}
+def ciHelper
+def DEFAULT_MODULE_NAME = 'nbi'
+pipeline {
+ agent { label 'pool' }
+ options { disableConcurrentBuilds() }
+ parameters {
+ // Core Gerrit / multibranch inputs
+ string(name: 'GERRIT_BRANCH', defaultValue: env.BRANCH_NAME ?: 'master', description: '')
+ string(name: 'GERRIT_PROJECT', defaultValue: 'osm/NBI', description: '')
+ string(name: 'GERRIT_REFSPEC', defaultValue: env.GERRIT_REFSPEC ?: '', description: '')
+ string(name: 'GERRIT_PATCHSET_REVISION', defaultValue: env.GERRIT_PATCHSET_REVISION ?: '', description: '')
+ string(name: 'PROJECT_URL_PREFIX', defaultValue: 'https://osm.etsi.org/gerrit', description: '')
+ string(name: 'ARTIFACTORY_SERVER', defaultValue: 'artifactory-osm', description: '')
+ string(name: 'DOCKER_ARGS', defaultValue: '', description: 'Extra docker args for docker run')
+
+ // Stage 3 parameters (mirroring central ci_stage_3.groovy)
+ // Core installer/E2E toggles
+ string(name: 'DOCKER_TAG', defaultValue: 'testing-daily', description: 'Tests image tag for opensourcemano/tests')
+ string(name: 'INSTALLER', defaultValue: 'Default', description: '')
+ string(name: 'OPENSTACK_BASE_IMAGE', defaultValue: 'ubuntu22.04', description: '')
+ string(name: 'OPENSTACK_OSM_FLAVOR', defaultValue: 'osm.sanity', description: '')
+
+ booleanParam(name: 'DO_BUILD', defaultValue: true, description: '')
+ booleanParam(name: 'DO_INSTALL', defaultValue: true, description: '')
+ booleanParam(name: 'DO_DOCKERPUSH', defaultValue: true, description: '')
+ booleanParam(name: 'DO_ROBOT', defaultValue: true, description: '')
+ string(name: 'ROBOT_TAG_NAME', defaultValue: 'sanity', description: 'Robot tag (sanity/regression/daily)')
+ string(name: 'ROBOT_PASS_THRESHOLD', defaultValue: '100.0', description: 'Pass threshold (%)')
+ string(name: 'ROBOT_UNSTABLE_THRESHOLD', defaultValue: '80.0', description: 'Unstable threshold (%)')
+ string(name: 'MODULE_NAME', defaultValue: 'NBI', description: 'Name of the module under test')
+
+ // Paths and configs
+ string(name: 'KUBECONFIG', defaultValue: '/home/jenkins/hive/kubeconfig.yaml', description: '')
+ string(name: 'CLOUDS', defaultValue: '/home/jenkins/hive/clouds.yaml', description: '')
+ string(name: 'ROBOT_VIM', defaultValue: '/home/jenkins/hive/robot-systest.cfg', description: '')
+ string(name: 'ROBOT_PORT_MAPPING_VIM', defaultValue: '/home/jenkins/hive/port-mapping-etsi-vim.yaml', description: '')
+ string(name: 'PROMETHEUS_CONFIG_VIM', defaultValue: '/home/jenkins/hive/etsi-vim-prometheus.json', description: '')
+ string(name: 'HIVE_VIM_1', defaultValue: '/home/jenkins/hive/openstack-etsi.rc', description: '')
+
+ // Feature flags and saves
+ booleanParam(name: 'TRY_JUJU_INSTALLATION', defaultValue: true, description: '')
+ booleanParam(name: 'TRY_OLD_SERVICE_ASSURANCE', defaultValue: false, description: '')
+ booleanParam(name: 'SAVE_CONTAINER_ON_FAIL', defaultValue: false, description: '')
+ booleanParam(name: 'SAVE_CONTAINER_ON_PASS', defaultValue: false, description: '')
+ booleanParam(name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS', defaultValue: true, description: '')
+ booleanParam(name: 'SAVE_ARTIFACTS_OVERRIDE', defaultValue: false, description: '')
+
+ // Optional publishing/flow controls
+ string(name: 'GPG_KEY_NAME', defaultValue: 'OSMETSI', description: '')
+ string(name: 'RELEASE', defaultValue: 'release', description: '')
+ string(name: 'REPO_DISTRO', defaultValue: 'unstable', description: '')
+ string(name: 'REPO_KEY_NAME', defaultValue: 'pubkey.asc', description: '')
+ string(name: 'COMMIT_ID', defaultValue: '', description: '')
+ string(name: 'UPSTREAM_JOB_NAME', defaultValue: '', description: '')
+ string(name: 'UPSTREAM_JOB_NUMBER', defaultValue: '', description: '')
+ string(name: 'UPSTREAM_SUFFIX', defaultValue: '-stage_2', description: '')
+ string(name: 'DOWNSTREAM_STAGE_NAME', defaultValue: 'osm-stage_4', description: '')
+ // Downstream params from NBI/Jenkinsfile NEW PIPELINE
+ booleanParam(name: 'TEST_INSTALL', defaultValue: false, description: 'Enable Stage 3/E2E in future')
+ string(name: 'IMAGENAME', defaultValue: 'opensourcemano/nbi', description: 'Image name for publish (reserved)')
+ }
+ environment {
+ MDG = "${params.GERRIT_PROJECT?.contains('/') ? params.GERRIT_PROJECT.split('/')[1] : params.GERRIT_PROJECT}"
+ CONTAINER_NAME = "${params.GERRIT_PROJECT}-${params.GERRIT_BRANCH}".toLowerCase()
+ TEST_IMAGE = 'overdrive3000/tox-osm:v1.6'
+ DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/'
+ }
+ stages {
+ stage('Prepare') { steps { sh 'env' } }
+
+ stage('Checkout') {
+ steps {
+ checkout scm
+ script {
+ sh "git fetch --tags"
+ if (params.GERRIT_REFSPEC?.trim()) { sh "git fetch origin ${params.GERRIT_REFSPEC}" }
+ if (params.GERRIT_PATCHSET_REVISION?.trim()) { sh "git checkout -f ${params.GERRIT_PATCHSET_REVISION}" }
+ sh "sudo git clean -dfx || git clean -dfx"
+ }
+ }
+ }
+
+ stage('Clone devops (central)') {
+ steps {
+ dir('devops') {
+ sh "git init ."
+ sh "git remote remove origin || true"
+ sh "git remote add origin ${params.PROJECT_URL_PREFIX}/osm/devops"
+ sh "git fetch --depth=1 origin ${params.GERRIT_BRANCH}"
+ sh "git checkout -f FETCH_HEAD"
+ }
+ }
+ }
+
+ stage('License Scan') {
+ steps {
+ script {
+ def isMergeJob = env.JOB_NAME?.contains('merge')
+ if (!isMergeJob) { sh 'devops/tools/license_scan.sh' } else { echo 'skip the scan for merge' }
+ }
+ }
+ }
+
+ stage('Prepare Test Image') {
+ steps {
+ script {
+ // Use shared test image from registry; no local build needed
+ sh "docker pull ${env.TEST_IMAGE} || true"
+ }
+ }
+ }
+
+ stage('Unit Tests') {
+ steps {
+ script {
+ if (!ciHelper) {
+ ciHelper = load 'devops/jenkins/ci-pipelines/ci_helper.groovy'
+ }
+ def UID = sh(returnStdout: true, script: 'id -u').trim()
+ def GID = sh(returnStdout: true, script: 'id -g').trim()
+ def common = "-v ${env.WORKSPACE}:/tests -e UID=${UID} -e GID=${GID} " + (params.DOCKER_ARGS ?: '')
+
+ stage('Test') {
+ sh """
+ docker run --rm ${common} \
+ ${env.TEST_IMAGE} \
+ /tests/devops-stages/stage-test.sh
+ """
+ if (fileExists('coverage.xml')) { cobertura coberturaReportFile: 'coverage.xml' }
+ if (fileExists('nosetests.xml')) { junit 'nosetests.xml' }
+ }
+
+ stage('Changelog') {
+ sh 'mkdir -p changelog'
+ sh """
+ docker run --rm ${common} \
+ ${env.TEST_IMAGE} \
+ /bin/sh -lc 'devops/tools/generatechangelog-pipeline.sh > /tests/changelog/changelog-${MDG}.html'
+ """
+ }
+ }
+ }
+ }
+
+ stage('Build & Push Image') {
+ when { expression { return params.DO_DOCKERPUSH } }
+ steps {
+ script {
+ def moduleName = (env.MDG ?: DEFAULT_MODULE_NAME).toLowerCase()
+
+ if (!params.GERRIT_BRANCH) {
+ error 'GERRIT_BRANCH is required to tag the Docker image'
+ }
+ def sanitizedBranchName = params.GERRIT_BRANCH
+ .toLowerCase()
+ .replaceAll('[^a-z0-9._-]', '-')
+ def baseTagPrefix = "osm-${sanitizedBranchName}"
+ def buildNumber = env.BUILD_NUMBER ?: '0'
+ def isMergeJob = env.JOB_NAME?.contains('merge')
+ // Remove promotion logic from this stage
+ def moduleTags = []
+ if (isMergeJob) {
+ moduleTags << "${baseTagPrefix}-merge-${buildNumber}"
+ } else {
+ moduleTags << "${baseTagPrefix}-patchset-${buildNumber}"
+ }
+
+ def imageName = params.IMAGENAME ?: "opensourcemano/${moduleName}"
+ def primaryLocalImage = "${imageName}:${moduleTags[0]}"
+
+ withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry',
+ usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
+ sh """
+ docker build -f Dockerfile.production -t ${primaryLocalImage} .
+ """
+ sh "docker login ${env.DOCKER_REGISTRY.split('/')[0]} -u ${USERNAME} -p ${PASSWORD}"
+ // Push build-scope tag(s) only. Promotion happens after tests.
+ moduleTags.each { tag ->
+ def localImage = "${imageName}:${tag}"
+ def remoteImage = "${env.DOCKER_REGISTRY}${imageName}:${tag}"
+ sh "docker tag ${localImage} ${remoteImage}"
+ sh "docker push ${remoteImage}"
+ }
+ // Stash the built image id for later promotion without rebuild
+ env.BUILT_IMAGE = primaryLocalImage
+ env.BUILT_TAG = moduleTags[0]
+ }
+ }
+ }
+ }
+
+ stage('E2E Test (robot)') {
+ when { expression { return params.DO_ROBOT && !env.JOB_NAME.contains('merge') } }
+ steps {
+ script {
+ def dowstreamJob = "osm-e2e/${params.GERRIT_BRANCH ?: 'master'}"
+ build job: dowstreamJob,
+ parameters: [
+ string(name: 'GERRIT_BRANCH', value: params.GERRIT_BRANCH ?: 'master'),
+ string(name: 'GERRIT_REFSPEC', value: params.GERRIT_REFSPEC ?: ''),
+ string(name: 'OPENSTACK_BASE_IMAGE', value: params.OPENSTACK_BASE_IMAGE),
+ string(name: 'OPENSTACK_OSM_FLAVOR', value: params.OPENSTACK_OSM_FLAVOR),
+ string(name: 'MODULE_NAME', value: params.MODULE_NAME ?: 'NBI'),
+ string(name: 'CONTAINER_NAME', value: env.BUILT_TAG),
+ booleanParam(name: 'DO_ROBOT', value: params.DO_ROBOT),
+ booleanParam(name: 'DO_INSTALL', value: params.DO_INSTALL),
+ booleanParam(name: 'SAVE_CONTAINER_ON_FAIL', value: params.SAVE_CONTAINER_ON_FAIL),
+ booleanParam(name: 'SAVE_CONTAINER_ON_PASS', value: params.SAVE_CONTAINER_ON_PASS)
+ ]
+ }
+ }
+ }
+ stage('Image Promotion') {
+ when { expression { return env.JOB_NAME?.contains('merge') } }
+ steps {
+ script {
+ def moduleName = (env.MDG ?: DEFAULT_MODULE_NAME).toLowerCase()
+ def sanitizedBranchName = params.GERRIT_BRANCH
+ .toLowerCase()
+ .replaceAll('[^a-z0-9._-]', '-')
+ def baseTagPrefix = "osm-${sanitizedBranchName}"
+ def targetTag = "${baseTagPrefix}-merge"
+ def imageName = params.IMAGENAME ?: "opensourcemano/${moduleName}"
+
+ withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry',
+ usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
+ sh "docker login ${env.DOCKER_REGISTRY.split('/')[0]} -u ${USERNAME} -p ${PASSWORD}"
+ def remoteImage = "${env.DOCKER_REGISTRY}${imageName}:${targetTag}"
+ sh "docker tag ${env.BUILT_IMAGE} ${remoteImage}"
+ sh "docker push ${remoteImage}"
+ }
+ }
+ }
+ }
+
+ /*
+ Promotion should be done in a separate downstream job after E2E success to avoid
+ */
+ }
+ post {
+ always {
+ cleanWs()
+ }
+ }
+}
diff --git a/devops-stages/new-pipeline/stage-lint.sh b/devops-stages/new-pipeline/stage-lint.sh
new file mode 100755
index 0000000..41f7645
--- /dev/null
+++ b/devops-stages/new-pipeline/stage-lint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Launching tox"
+TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto -e black,flake8,pylint,safety
+
diff --git a/devops-stages/new-pipeline/stage-test.sh b/devops-stages/new-pipeline/stage-test.sh
new file mode 100755
index 0000000..09eacb1
--- /dev/null
+++ b/devops-stages/new-pipeline/stage-test.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Launching tox"
+TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto -e cover
+