dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 7, unit: 'HOURS')
  29         timestamps()
  30         checkoutToSubdirectory('src')
  31     }
  32
  33     environment {
  34         SOURCEDIR = 'src'
  35         // will also need to change notification section below
  36         PATCHDIR = 'out'
  37         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
  38         YETUS='yetus'
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION='rel/0.11.1'
  41     }
  42
  43     parameters {
  44         booleanParam(name: 'DEBUG',
  45                defaultValue: false,
  46                description: 'Print extra outputs for debugging the jenkins job and yetus')
  47     }
  48
  49     stages {
  50         stage ('install yetus') {
  51             steps {
  52                 dir("${WORKSPACE}/${YETUS}") {
  53                     checkout([
  54                         $class: 'GitSCM',
  55                         branches: [[name: "${env.YETUS_VERSION}"]],
  56                         userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
  57                     )
  58                 }
  59             }
  60         }
  61
  62         stage ('precommit-run') {
  63             steps {
  64                 withCredentials(
  65                     [usernamePassword(credentialsId: 'apache-hbase-at-github.com',
  66                                   passwordVariable: 'GITHUB_PASSWORD',
  67                                   usernameVariable: 'GITHUB_USER'),
  68                     usernamePassword(credentialsId: 'hbaseqa-at-asf-jira',
  69                                         passwordVariable: 'JIRA_PASSWORD',
  70                                         usernameVariable: 'JIRA_USER')]) {
  71                         sh '''#!/usr/bin/env bash
  72                         set -e
  73                         TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
  74                         # this must be clean for every run
  75                         if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
  76                           rm -rf "${WORKSPACE}/${PATCHDIR}"
  77                         fi
  78                         mkdir -p "${WORKSPACE}/${PATCHDIR}"
  79
  80                         ## Checking on H* machine nonsense
  81                         echo "JAVA_HOME: ${JAVA_HOME}"
  82                         ls -l "${JAVA_HOME}" || true
  83                         echo "MAVEN_HOME: ${MAVEN_HOME}"
  84                         echo "maven version:"
  85                         mvn --offline --version  || true
  86                         echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/"
  87                         mkdir "${PATCHDIR}/machine"
  88                         cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true
  89                         cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true
  90                         cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true
  91                         cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true
  92                         df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true
  93                         ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true
  94                         ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true
  95                         lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true
  96                         lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true
  97                         cat /proc/loadavg >"${output}/loadavg" 2>&1 || true
  98                         ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true
  99                         ## /H*
 100
 101                         # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
 102                         # Otherwise exit, because we don't want HBase to do a
 103                         # full build.  We wouldn't normally do this check for smaller
 104                         # projects. :)
 105                         if [[ -z "${CHANGE_URL}" ]]; then
 106                             echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
 107                             exit 0
 108                         fi
 109                         # enable debug output for yetus
 110                         if [[ "true" = "${DEBUG}" ]]; then
 111                             YETUS_ARGS+=("--debug")
 112                         fi
 113                         # If we're doing docker, make sure we don't accidentally pollute the image with a host java path
 114                         if [ -n "${JAVA_HOME}" ]; then
 115                           unset JAVA_HOME
 116                         fi
 117                         YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
 118                         # where the source is located
 119                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 120                         # our project defaults come from a personality file
 121                         # which will get loaded automatically by setting the project name
 122                         YETUS_ARGS+=("--project=hbase")
 123                         # lots of different output formats
 124                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
 125                         YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
 126                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 127                         # enable writing back to Github
 128                         YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
 129                         YETUS_ARGS+=(--github-user=${GITHUB_USER})
 130                         # enable writing back to ASF JIRA
 131                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
 132                         YETUS_ARGS+=(--jira-user="${JIRA_USER}")
 133                         # auto-kill any surefire stragglers during unit test runs
 134                         YETUS_ARGS+=("--reapermode=kill")
 135                         YETUS_ARGS+=("--multijdktests=compile")
 136                         # set relatively high limits for ASF machines
 137                         # changing these to higher values may cause problems
 138                         # with other jobs on systemd-enabled machines
 139                         YETUS_ARGS+=("--proclimit=10000")
 140                         YETUS_ARGS+=("--dockermemlimit=20g")
 141                         # -1 findbugs issues that show up prior to the patch being applied
 142                         YETUS_ARGS+=("--findbugs-strict-precheck")
 143                         # rsync these files back into the archive dir
 144                         YETUS_ARGS+=("--archive-list=rat.txt")
 145                         # URL for user-side presentation in reports and such to our artifacts
 146                         # (needs to match the archive bits below)
 147                         YETUS_ARGS+=("--build-url-artifacts=artifact/out")
 148                         # plugins to enable
 149                         YETUS_ARGS+=("--plugins=all")
 150                         # don't let these tests cause -1s because we aren't really paying that
 151                         # much attention to them
 152                         YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests")
 153                         # run in docker mode and specifically point to our
 154                         # Dockerfile since we don't want to use the auto-pulled version.
 155                         YETUS_ARGS+=("--docker")
 156                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 157                         YETUS_ARGS+=("--mvn-custom-repos")
 158                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64")
 159                         YETUS_ARGS+=("--findbugs-home=/usr")
 160                         YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
 161                         YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
 162                         YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 163                         YETUS_ARGS+=("--quick-hadoopcheck")
 164                         YETUS_ARGS+=("--skip-errorprone")
 165                         # effectively treat dev-support as a custom maven module
 166                         YETUS_ARGS+=("--skip-dirs=dev-support")
 167                         # help keep the ASF boxes clean
 168                         YETUS_ARGS+=("--sentinel")
 169                         # use emoji vote so it is easier to find the broken line
 170                         YETUS_ARGS+=("--github-use-emoji-vote")
 171                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
 172                         '''
 173                 }
 174             }
 175         }
 176
 177     }
 178
 179     post {
 180         always {
 181           script {
 182             // Yetus output
 183             archiveArtifacts "${env.PATCHDIR}/**"
 184             // Publish the HTML report so that it can be looked at
 185             // Has to be relative to WORKSPACE.
 186             publishHTML (target: [
 187                           allowMissing: true,
 188                           keepAll: true,
 189                           alwaysLinkToLastBuild: true,
 190                           // Has to be relative to WORKSPACE
 191                           reportDir: "${env.PATCHDIR}",
 192                           reportFiles: 'report.html',
 193                           reportName: 'Yetus Report'
 194             ])
 195             // Publish JUnit results
 196             try {
 197                 junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
 198             } catch(e) {
 199                 echo 'junit processing: ' + e.toString()
 200             }
 201           }
 202         }
 203
 204         // Jenkins pipeline jobs fill slaves on PRs without this :(
 205         cleanup() {
 206             script {
 207                 sh '''
 208                     # See YETUS-764
 209                     if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
 210                       echo "test-patch process appears to still be running: killing"
 211                       kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
 212                       sleep 10
 213                     fi
 214                     if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
 215                       echo "test-patch container appears to still be running: killing"
 216                       docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
 217                     fi
 218                     # See HADOOP-13951
 219                     chmod -R u+rxw "${WORKSPACE}"
 220                     '''
 221                 deleteDir()
 222             }
 223         }
 224     }
 225 }