mirror of
https://github.com/corda/corda.git
synced 2024-12-23 14:52:29 +00:00
Merge pull request #6729 from corda/INFRA-683-os-47-linear-builds-parallel-stages
INFRA-683: Switching linear regression build on two agents
This commit is contained in:
Waldemar Żurowski
GitHub
commit
da73cbf2a5
310
.ci/dev/regression/Jenkinsfile
vendored
310
.ci/dev/regression/Jenkinsfile
vendored
@ -1,29 +1,21 @@
|
||||
#!groovy
|
||||
/**
|
||||
* Jenkins pipeline to build Corda OS release branches and tags
|
||||
* Jenkins pipeline to build Corda OS release branches and tags.
|
||||
* PLEASE NOTE: we DO want to run a build for each commit!!!
|
||||
*/
|
||||
|
||||
/**
|
||||
* Kill already started job.
|
||||
* Assume new commit takes precendence and results from previous
|
||||
* unfinished builds are not required.
|
||||
* This feature doesn't play well with disableConcurrentBuilds() option
|
||||
*/
|
||||
@Library('corda-shared-build-pipeline-steps')
|
||||
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
|
||||
|
||||
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
|
||||
|
||||
/**
|
||||
* Sense environment
|
||||
*/
|
||||
boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?<!_JDK11)$/)
|
||||
boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
|
||||
boolean isReleaseCandidate = (env.TAG_NAME =~ /^(release-.*(RC|HC).*(?<!_JDK11))$/)
|
||||
|
||||
/*
|
||||
** calculate the stage for NexusIQ evaluation
|
||||
** * build for snapshots
|
||||
** * stage-release: for release candidates and for health checks
|
||||
** * operate: for final release
|
||||
** * release: for GA release
|
||||
*/
|
||||
def nexusDefaultIqStage = "build"
|
||||
if (isReleaseTag) {
|
||||
@ -47,40 +39,64 @@ def nexusIqStageChoices = [nexusDefaultIqStage].plus(
|
||||
'operate'
|
||||
].minus([nexusDefaultIqStage]))
|
||||
|
||||
/**
|
||||
* Common Gradle arguments for all Gradle executions
|
||||
*/
|
||||
String COMMON_GRADLE_PARAMS = [
|
||||
'--no-daemon',
|
||||
'--stacktrace',
|
||||
'--info',
|
||||
'-Pcompilation.warningsAsErrors=false',
|
||||
'-Ptests.failFast=true',
|
||||
].join(' ')
|
||||
|
||||
pipeline {
|
||||
agent { label 'k8s' }
|
||||
agent { label 'standard' }
|
||||
|
||||
/*
|
||||
* List options in alphabetical order
|
||||
*/
|
||||
options {
|
||||
timestamps()
|
||||
disableConcurrentBuilds()
|
||||
timeout(time: 3, unit: 'HOURS')
|
||||
buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
|
||||
parallelsAlwaysFailFast()
|
||||
timeout(time: 6, unit: 'HOURS')
|
||||
timestamps()
|
||||
}
|
||||
|
||||
parameters {
|
||||
choice choices: nexusIqStageChoices, description: 'NexusIQ stage for code evaluation', name: 'nexusIqStage'
|
||||
}
|
||||
|
||||
/*
|
||||
* List environment variables in alphabetical order
|
||||
*/
|
||||
environment {
|
||||
DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}"
|
||||
DOCKER_URL = "https://index.docker.io/v1/"
|
||||
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
|
||||
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
|
||||
ARTIFACTORY_BUILD_NAME = "Corda :: Publish :: Publish Release to Artifactory :: ${env.BRANCH_NAME}"
|
||||
ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
|
||||
ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::")
|
||||
CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
|
||||
CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
|
||||
CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
|
||||
DOCKER_URL = "https://index.docker.io/v1/"
|
||||
}
|
||||
|
||||
stages {
|
||||
/*
|
||||
* Temporarily disable Sonatype checks for regression builds
|
||||
*/
|
||||
stage('Sonatype Check') {
|
||||
when {
|
||||
expression { isReleaseTag }
|
||||
}
|
||||
stage('Compile') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'clean',
|
||||
'jar'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
|
||||
stage('Stash') {
|
||||
steps {
|
||||
stash name: 'compiled', useDefaultExcludes: false
|
||||
}
|
||||
}
|
||||
stage('Sonatype Check') {
|
||||
steps {
|
||||
sh "./gradlew --no-daemon clean jar"
|
||||
script {
|
||||
sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
|
||||
/* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
|
||||
@ -98,82 +114,130 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy Nodes') {
|
||||
steps {
|
||||
sh "./gradlew --no-daemon jar deployNodes"
|
||||
}
|
||||
}
|
||||
|
||||
stage('Generate Build Image') {
|
||||
steps {
|
||||
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
|
||||
sh "./gradlew " +
|
||||
"-Dkubenetize=true " +
|
||||
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
|
||||
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
|
||||
"-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
|
||||
"-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
|
||||
"-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
|
||||
" clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest " +
|
||||
" preAllocateForAllParallelSlowIntegrationTest preAllocateForAllParallelSmokeTest " +
|
||||
" pushBuildImage --stacktrace"
|
||||
}
|
||||
sh "kubectl auth can-i get pods"
|
||||
}
|
||||
}
|
||||
|
||||
stage('Testing phase') {
|
||||
stage('All Tests') {
|
||||
parallel {
|
||||
stage('Unit Test') {
|
||||
steps {
|
||||
sh "./gradlew " +
|
||||
"-DbuildId=\"\${BUILD_ID}\" " +
|
||||
"-Dkubenetize=true " +
|
||||
"-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
|
||||
"-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
|
||||
"-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
|
||||
"-Dgit.branch=\"\${GIT_BRANCH}\" " +
|
||||
"-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
|
||||
" allParallelUnitTest --stacktrace"
|
||||
stage('Another agent') {
|
||||
agent {
|
||||
label 'standard'
|
||||
}
|
||||
options {
|
||||
skipDefaultCheckout true
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: '**/*.log', fingerprint: false
|
||||
junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
|
||||
/*
|
||||
* Copy all JUnit results files into a single top level directory.
|
||||
* This is necessary to stop the allure plugin from hitting out
|
||||
* of memory errors due to being passed many directories with
|
||||
* long paths.
|
||||
*
|
||||
* File names are pre-pended with a prefix when
|
||||
* copied to avoid collisions between files where the same test
|
||||
* classes have run on multiple agents.
|
||||
*/
|
||||
fileOperations([fileCopyOperation(
|
||||
includes: '**/build/test-results/**/*.xml',
|
||||
targetLocation: 'allure-input',
|
||||
flattenFiles: true,
|
||||
renameFiles: true,
|
||||
sourceCaptureExpression: '.*/([^/]+)$',
|
||||
targetNameExpression: 'other-agent-$1')])
|
||||
stash name: 'allure-input', includes: 'allure-input/**', useDefaultExcludes: false
|
||||
}
|
||||
cleanup {
|
||||
deleteDir() /* clean up our workspace */
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('Unstash') {
|
||||
steps {
|
||||
unstash 'compiled'
|
||||
}
|
||||
}
|
||||
stage('Recompile') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'jar'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
stage('Unit Test') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'test'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
stage('Smoke Test') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'smokeTest'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
stage('Slow Integration Test') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'slowIntegrationTest'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Integration Test') {
|
||||
steps {
|
||||
sh "./gradlew " +
|
||||
"-DbuildId=\"\${BUILD_ID}\" " +
|
||||
"-Dkubenetize=true " +
|
||||
"-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
|
||||
"-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
|
||||
"-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
|
||||
"-Dgit.branch=\"\${GIT_BRANCH}\" " +
|
||||
"-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
|
||||
" allParallelIntegrationTest --stacktrace"
|
||||
stage('Same agent') {
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: '**/*.log', fingerprint: false
|
||||
junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
|
||||
/*
|
||||
* Copy all JUnit results files into a single top level directory.
|
||||
* This is necessary to stop the allure plugin from hitting out
|
||||
* of memory errors due to being passed many directories with
|
||||
* long paths.
|
||||
*
|
||||
* File names are pre-pended with a prefix when
|
||||
* copied to avoid collisions between files where the same test
|
||||
* classes have run on multiple agents.
|
||||
*/
|
||||
fileOperations([fileCopyOperation(
|
||||
includes: '**/build/test-results/**/*.xml',
|
||||
targetLocation: 'allure-input',
|
||||
flattenFiles: true,
|
||||
renameFiles: true,
|
||||
sourceCaptureExpression: '.*/([^/]+)$',
|
||||
targetNameExpression: 'same-agent-$1')])
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Smoke Test') {
|
||||
steps {
|
||||
sh "./gradlew " +
|
||||
"-DbuildId=\"\${BUILD_ID}\" " +
|
||||
"-Dkubenetize=true " +
|
||||
"-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
|
||||
"-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
|
||||
"-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
|
||||
"-Dgit.branch=\"\${GIT_BRANCH}\" " +
|
||||
"-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
|
||||
" allParallelSmokeTest --stacktrace"
|
||||
}
|
||||
}
|
||||
stage('Slow Integration Test') {
|
||||
steps {
|
||||
sh "./gradlew " +
|
||||
"-DbuildId=\"\${BUILD_ID}\" " +
|
||||
"-Dkubenetize=true " +
|
||||
"-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
|
||||
"-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
|
||||
"-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
|
||||
"-Dgit.branch=\"\${GIT_BRANCH}\" " +
|
||||
"-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
|
||||
" allParallelSlowIntegrationTest --stacktrace"
|
||||
stages {
|
||||
stage('Integration Test') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'integrationTest'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy Node') {
|
||||
steps {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'deployNode'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -211,14 +275,19 @@ pipeline {
|
||||
|
||||
stage('Publish Release to Docker Hub') {
|
||||
when {
|
||||
expression { !isInternalRelease && isReleaseTag }
|
||||
expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate}
|
||||
}
|
||||
steps {
|
||||
withCredentials([
|
||||
usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials',
|
||||
usernameVariable: 'DOCKER_USERNAME',
|
||||
passwordVariable: 'DOCKER_PASSWORD')]) {
|
||||
sh "./gradlew pushOfficialImages"
|
||||
passwordVariable: 'DOCKER_PASSWORD')
|
||||
]) {
|
||||
sh script: [
|
||||
'./gradlew',
|
||||
COMMON_GRADLE_PARAMS,
|
||||
'pushOfficialImages'
|
||||
].join(' ')
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -226,31 +295,12 @@ pipeline {
|
||||
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
|
||||
junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
|
||||
|
||||
script {
|
||||
try {
|
||||
/*
|
||||
* Copy all JUnit results files into a single top level directory.
|
||||
* This is necessary to stop the allure plugin from hitting out
|
||||
* of memory errors due to being passed many directories with
|
||||
* long paths.
|
||||
*
|
||||
* File names are pre-pended with the pod number when
|
||||
* copied to avoid collisions between files where the same test
|
||||
* classes have run on multiple pods.
|
||||
*/
|
||||
fileOperations([fileCopyOperation(
|
||||
includes: '**/test-results-xml/**/test-runs/test-reports/**',
|
||||
targetLocation: 'allure-input',
|
||||
flattenFiles: true,
|
||||
renameFiles: true,
|
||||
sourceCaptureExpression: '.*test-results-xml/.*-([\\d]+)/.*/([^/]+)$',
|
||||
targetNameExpression: '$1-$2')])
|
||||
unstash 'allure-input'
|
||||
allure includeProperties: false,
|
||||
jdk: '',
|
||||
results: [[path: '**/allure-input']]
|
||||
jdk: '',
|
||||
results: [[path: '**/allure-input']]
|
||||
} catch (err) {
|
||||
echo("Allure report generation failed: $err")
|
||||
|
||||
@ -267,7 +317,7 @@ pipeline {
|
||||
// Comparing the dates of the previous and current builds achieves this,
|
||||
// i.e. we will only send an email for the first build on a given day.
|
||||
def prevBuildDate = new Date(
|
||||
currentBuild?.previousBuild.timeInMillis ?: 0).clearTime()
|
||||
currentBuild.previousBuild?.timeInMillis ?: 0).clearTime()
|
||||
def currentBuildDate = new Date(
|
||||
currentBuild.timeInMillis).clearTime()
|
||||
|
||||
@ -278,6 +328,8 @@ pipeline {
|
||||
statusSymbol = '\u2705'
|
||||
break;
|
||||
case 'UNSTABLE':
|
||||
statusSymbol = '\u26A0'
|
||||
break;
|
||||
case 'FAILURE':
|
||||
statusSymbol = '\u274c'
|
||||
break;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user