Jenkins move all compile jobs to dedicated pipeline (#10149)

This commit is contained in:
Daniel Agar 2018-08-04 12:23:46 -04:00 committed by GitHub
parent 1d4ef1e6fa
commit a889ad8e8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 244 additions and 281 deletions

242
.ci/Jenkinsfile-compile Normal file
View File

@ -0,0 +1,242 @@
#!/usr/bin/env groovy
pipeline {
agent none
stages {
stage('Build') {
steps {
script {
def build_nodes = [:]
def docker_images = [
arch: "px4io/px4-dev-base-archlinux:2018-07-19",
armhf: "px4io/px4-dev-armhf:2018-07-19",
base: "px4io/px4-dev-base:2018-07-19",
clang: "px4io/px4-dev-clang:2018-07-19",
nuttx: "px4io/px4-dev-nuttx:2018-07-19",
ros: "px4io/px4-dev-ros:2018-07-19",
rpi: "px4io/px4-dev-raspi:2018-07-19",
snapdragon: "lorenzmeier/px4-dev-snapdragon:2017-12-29"
]
// special builds:
// fmu-v2_{default, lpe} and fmu-v3_{default, rtps}
// bloaty compare to last successful master build
build_nodes["px4fmu-v2"] = {
node {
stage("Build Test px4fmu-v2") {
docker.image(docker_images.nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage("px4fmu-v2") {
try {
checkout(scm)
sh('export')
sh('make distclean')
sh('ccache -z')
sh('git fetch --tags')
sh('make nuttx_px4io-v2_default')
sh('make nuttx_px4io-v2_default bloaty_symbols')
sh('make nuttx_px4io-v2_default bloaty_compileunits')
sh('make nuttx_px4io-v2_default bloaty_compare_master')
sh('make nuttx_px4fmu-v2_default')
sh('make nuttx_px4fmu-v2_default bloaty_symbols')
sh('make nuttx_px4fmu-v2_default bloaty_compileunits')
sh('make nuttx_px4fmu-v2_default bloaty_inlines')
sh('make nuttx_px4fmu-v2_default bloaty_templates')
sh('make nuttx_px4fmu-v2_default bloaty_compare_master')
sh('make nuttx_px4fmu-v2_lpe')
sh('make nuttx_px4fmu-v2_test')
sh('make nuttx_px4fmu-v3_default')
sh('make nuttx_px4fmu-v3_default bloaty_compare_master')
sh('make nuttx_px4fmu-v3_rtps')
sh('make sizes')
sh('ccache -s')
archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
}
catch (exc) {
throw (exc)
}
finally {
sh('make distclean')
}
}
}
}
}
}
// MAC OSX posix_sitl_default
build_nodes["posix_sitl_default (OSX)"] = {
node("mac") {
withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
stage("sitl (OSX)") {
try {
checkout(scm)
sh('export')
sh('make distclean')
sh('ccache -z')
sh('make posix_sitl_default')
sh('ccache -s')
}
catch (exc) {
throw (exc)
}
finally {
sh('make distclean')
}
}
}
}
}
// MAC OSX nuttx_px4fmu-v4pro_default
build_nodes["px4fmu-v4pro_default (OSX)"] = {
node("mac") {
withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
stage("px4fmu-v4pro (OSX)") {
try {
checkout(scm)
sh('export')
sh('make distclean')
sh('ccache -z')
sh('make nuttx_px4fmu-v4pro_default')
sh('ccache -s')
}
catch (exc) {
throw (exc)
}
finally {
sh('make distclean')
}
}
}
}
}
// docker builds:
def arch_builds = [
target: ["posix_sitl_default"],
image: docker_images.arch,
archive: false
]
def armhf_builds = [
target: ["posix_ocpoc_ubuntu"],
image: docker_images.armhf,
archive: false
]
def base_builds = [
target: ["posix_sitl_rtps"],
image: docker_images.base,
archive: false
]
def nuttx_builds_archive = [
target: ["px4fmu-v4_default", "px4fmu-v4pro_default", "px4fmu-v5_default", "aerofc-v1_default",
"aerocore2_default", "auav-x21_default", "crazyflie_default", "mindpx-v2_default",
"nxphlite-v3_default", "tap-v1_default", "omnibus-f4sd_default"],
image: docker_images.nuttx,
archive: true
]
def nuttx_builds_other = [
target: ["px4-same70xplained-v1_default", "px4-stm32f4discovery_default", "px4cannode-v1_default",
"px4esc-v1_default", "px4nucleoF767ZI-v1_default", "s2740vc-v1_default"],
image: docker_images.nuttx,
archive: false
]
def rpi_builds = [
target: ["posix_rpi_cross", "posix_bebop_default"],
image: docker_images.rpi,
archive: false
]
def snapdragon_builds = [
target: ["qurt_eagle_default", "posix_eagle_default"],
image: docker_images.snapdragon,
archive: false
]
def docker_builds = [
arch_builds, armhf_builds, base_builds, nuttx_builds_archive, nuttx_builds_other,
rpi_builds, snapdragon_builds
]
for (def build_type = 0; build_type < docker_builds.size(); build_type++) {
for (def build_target = 0; build_target < docker_builds[build_type].target.size(); build_target++) {
build_nodes.put(docker_builds[build_type].target[build_target],
createBuildNode(docker_builds[build_type].archive, docker_builds[build_type].image, docker_builds[build_type].target[build_target])
)
}
}
parallel build_nodes
} // script
} // steps
} // stage Build
// TODO: actually upload artifacts to S3
stage('S3 Upload') {
agent {
docker { image 'px4io/px4-dev-base:2018-07-19' }
}
options {
skipDefaultCheckout()
}
when {
anyOf {
branch 'master'
branch 'beta'
branch 'stable'
branch 'pr-jenkins' // for testing
}
}
steps {
sh 'echo "uploading to S3"'
}
}
} // stages
environment {
CCACHE_DIR = '/tmp/ccache'
CI = true
}
options {
buildDiscarder(logRotator(numToKeepStr: '10', artifactDaysToKeepStr: '30'))
timeout(time: 60, unit: 'MINUTES')
}
}
def createBuildNode(Boolean archive, String docker_image, String target) {
return {
node {
docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') {
docker.image(docker_image).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage(target) {
try {
sh('export')
checkout(scm)
sh('make distclean')
sh('git fetch --tags')
sh('ccache -z')
sh('make ' + target)
sh('ccache -s')
sh('make sizes')
if (archive) {
archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true)
}
}
catch (exc) {
throw (exc)
}
finally {
sh('make distclean')
}
}
}
}
}
}
}

283
Jenkinsfile vendored
View File

@ -2,141 +2,6 @@ pipeline {
agent none
stages {
stage('Build') {
steps {
script {
def builds = [:]
def docker_base = "px4io/px4-dev-base:2018-07-19"
def docker_nuttx = "px4io/px4-dev-nuttx:2018-07-19"
def docker_ros = "px4io/px4-dev-ros:2018-07-19"
def docker_rpi = "px4io/px4-dev-raspi:2018-07-19"
def docker_armhf = "px4io/px4-dev-armhf:2018-07-19"
def docker_arch = "px4io/px4-dev-base-archlinux:2018-07-19"
def docker_snapdragon = "lorenzmeier/px4-dev-snapdragon:2017-12-29"
def docker_clang = "px4io/px4-dev-clang:2018-07-19"
// fmu-v2_{default, lpe} and fmu-v3_{default, rtps}
// bloaty compare to last successful master build
builds["px4fmu-v2"] = {
node {
stage("Build Test px4fmu-v2") {
docker.image(docker_nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage("px4fmu-v2") {
checkout scm
sh "export"
sh "make distclean"
sh "ccache -z"
sh "git fetch --tags"
sh "make nuttx_px4io-v2_default"
sh "make nuttx_px4io-v2_default bloaty_symbols"
sh "make nuttx_px4io-v2_default bloaty_compileunits"
sh "make nuttx_px4io-v2_default bloaty_compare_master"
sh "make nuttx_px4fmu-v2_default"
sh "make nuttx_px4fmu-v2_default bloaty_symbols"
sh "make nuttx_px4fmu-v2_default bloaty_compileunits"
sh "make nuttx_px4fmu-v2_default bloaty_inlines"
sh "make nuttx_px4fmu-v2_default bloaty_templates"
sh "make nuttx_px4fmu-v2_default bloaty_compare_master"
sh "make nuttx_px4fmu-v2_lpe"
sh "make nuttx_px4fmu-v2_test"
sh "make nuttx_px4fmu-v3_default"
sh "make nuttx_px4fmu-v3_rtps"
sh "make sizes"
sh "ccache -s"
archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
sh "make distclean"
}
}
}
}
}
// nuttx default targets that are archived and uploaded to s3
for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1", "aerocore2", "av-x-v1", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1", "omnibus-f4sd"]) {
def node_name = "${option}"
builds[node_name] = createBuildNodeArchive(docker_nuttx, "${node_name}_default")
}
// other nuttx default targets
for (def option in ["px4-same70xplained-v1", "px4-stm32f4discovery", "px4cannode-v1", "px4esc-v1", "px4nucleoF767ZI-v1", "s2740vc-v1"]) {
def node_name = "${option}"
builds[node_name] = createBuildNode(docker_nuttx, "${node_name}_default")
}
builds["sitl_rtps"] = createBuildNode(docker_base, 'posix_sitl_rtps')
builds["sitl (GCC 7)"] = createBuildNode(docker_arch, 'posix_sitl_default')
builds["rpi"] = createBuildNode(docker_rpi, 'posix_rpi_cross')
builds["bebop"] = createBuildNode(docker_rpi, 'posix_bebop_default')
builds["ocpoc"] = createBuildNode(docker_armhf, 'posix_ocpoc_ubuntu')
// snapdragon (eagle_default)
builds["eagle (linux)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'posix_eagle_default')
builds["eagle (qurt)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'qurt_eagle_default')
// posix_sitl_default with package
builds["sitl"] = {
node {
stage("Build Test sitl") {
docker.image(docker_ros).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE') {
stage("sitl") {
checkout scm
sh "export"
sh "make distclean"
sh "ccache -z"
sh "make posix_sitl_default"
sh "make posix_sitl_default sitl_gazebo"
sh "make posix_sitl_default package"
sh "ccache -s"
stash name: "px4_sitl_package", includes: "build/posix_sitl_default/*.bz2"
sh "make distclean"
}
}
}
}
}
// MAC OS posix_sitl_default
builds["sitl (OSX)"] = {
node("mac") {
withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
stage("sitl (OSX)") {
checkout scm
sh "export"
sh "make distclean"
sh "ccache -z"
sh "make posix_sitl_default"
sh "ccache -s"
sh "make distclean"
}
}
}
}
// MAC OS nuttx_px4fmu-v4pro_default
builds["px4fmu-v4pro (OSX)"] = {
node("mac") {
withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
stage("px4fmu-v4pro (OSX)") {
checkout scm
sh "export"
sh "make distclean"
sh "ccache -z"
sh "make nuttx_px4fmu-v4pro_default"
sh "ccache -s"
sh "make distclean"
}
}
}
}
parallel builds
} // script
} // steps
} // stage Builds
stage('Test') {
parallel {
@ -178,6 +43,7 @@ pipeline {
branch 'master'
branch 'beta'
branch 'stable'
branch 'pr-jenkins' // for testing
}
}
}
@ -225,6 +91,7 @@ pipeline {
branch 'master'
branch 'beta'
branch 'stable'
branch 'pr-jenkins' // for testing
}
}
}
@ -324,70 +191,6 @@ pipeline {
}
}
stage('ROS MC mission box (EKF2)') {
agent {
docker {
image 'px4io/px4-dev-ros:2018-07-19'
args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE'
}
}
options {
skipDefaultCheckout()
}
steps {
sh 'export'
sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo'
unstash 'px4_sitl_package'
sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2'
sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris'
sh 'px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py `find . -name *.ulg -print -quit`'
}
post {
always {
sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
archiveArtifacts '.ros/**/*.pdf'
archiveArtifacts '.ros/**/*.csv'
deleteDir()
}
failure {
sh 'ls -a'
archiveArtifacts '.ros/**/*.ulg'
archiveArtifacts '.ros/**/rosunit-*.xml'
archiveArtifacts '.ros/**/rostest-*.log'
}
}
}
stage('ROS MC mission box (LPE)') {
agent {
docker {
image 'px4io/px4-dev-ros:2018-07-19'
args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE'
}
}
options {
skipDefaultCheckout()
}
steps {
sh 'export'
sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo'
unstash 'px4_sitl_package'
sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2'
sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris est:=lpe'
}
post {
always {
sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
deleteDir()
}
failure {
sh 'ls -a'
archiveArtifacts '.ros/**/*.ulg'
archiveArtifacts '.ros/**/rosunit-*.xml'
archiveArtifacts '.ros/**/rostest-*.log'
}
}
}
}
}
@ -450,25 +253,6 @@ pipeline {
}
}
// TODO: actually upload artifacts to S3
stage('S3 Upload') {
agent {
docker { image 'px4io/px4-dev-base:2018-07-19' }
}
options {
skipDefaultCheckout()
}
when {
anyOf {
branch 'master'
branch 'beta'
branch 'stable'
}
}
steps {
sh 'echo "uploading to S3"'
}
}
} // stages
environment {
@ -480,66 +264,3 @@ pipeline {
timeout(time: 60, unit: 'MINUTES')
}
}
def createBuildNode(String docker_repo, String target) {
return {
node {
docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage(target) {
sh('export')
checkout scm
sh('make distclean')
sh('git fetch --tags')
sh('ccache -z')
sh('make ' + target)
sh('ccache -s')
sh('make sizes')
sh('make distclean')
}
}
}
}
}
def createBuildNodeArchive(String docker_repo, String target) {
return {
node {
docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage(target) {
sh('export')
checkout scm
sh('make distclean')
sh('git fetch --tags')
sh('ccache -z')
sh('make ' + target)
sh('ccache -s')
sh('make sizes')
archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true)
sh('make distclean')
}
}
}
}
}
def createBuildNodeDockerLogin(String docker_repo, String docker_credentials, String target) {
return {
node {
docker.withRegistry('https://registry.hub.docker.com', docker_credentials) {
docker.image(docker_repo).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
stage(target) {
sh('export')
checkout scm
sh('make distclean')
sh('git fetch --tags')
sh('ccache -z')
sh('make ' + target)
sh('ccache -s')
sh('make sizes')
sh('make distclean')
}
}
}
}
}
}