dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 7, unit: 'HOURS')
  29         timestamps()
  30         checkoutToSubdirectory('src')
  31     }
  32
  33     environment {
  34         SOURCEDIR = 'src'
  35         // will also need to change notification section below
  36         PATCHDIR = 'out'
  37         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
  38         YETUS='yetus'
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION='rel/0.10.0'
  41     }
  42
  43     parameters {
  44         booleanParam(name: 'DEBUG',
  45                defaultValue: false,
  46                description: 'Print extra outputs for debugging the jenkins job and yetus')
  47     }
  48
  49     stages {
  50         stage ('install yetus') {
  51             steps {
  52                 dir("${WORKSPACE}/${YETUS}") {
  53                     checkout([
  54                         $class: 'GitSCM',
  55                         branches: [[name: "${env.YETUS_VERSION}"]],
  56                         userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
  57                     )
  58                 }
  59             }
  60         }
  61
  62         stage ('precommit-run') {
  63             steps {
  64                 withCredentials(
  65                     [usernamePassword(credentialsId: 'apache-hbase-at-github.com',
  66                                   passwordVariable: 'GITHUB_PASSWORD',
  67                                   usernameVariable: 'GITHUB_USER'),
  68                     usernamePassword(credentialsId: 'hbaseqa-at-asf-jira',
  69                                         passwordVariable: 'JIRA_PASSWORD',
  70                                         usernameVariable: 'JIRA_USER')]) {
  71                         sh '''#!/usr/bin/env bash
  72                         set -e
  73                         TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
  74                         # this must be clean for every run
  75                         if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
  76                           rm -rf "${WORKSPACE}/${PATCHDIR}"
  77                         fi
  78                         mkdir -p "${WORKSPACE}/${PATCHDIR}"
  79
  80                         ## Checking on H* machine nonsense
  81                         echo "JAVA_HOME: ${JAVA_HOME}"
  82                         ls -l "${JAVA_HOME}" || true
  83                         echo "MAVEN_HOME: ${MAVEN_HOME}"
  84                         echo "maven version:"
  85                         mvn --offline --version  || true
  86                         echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/"
  87                         mkdir "${PATCHDIR}/machine"
  88                         cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true
  89                         cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true
  90                         cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true
  91                         cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true
  92                         df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true
  93                         ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true
  94                         ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true
  95                         lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true
  96                         lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true
  97                         cat /proc/loadavg >"${output}/loadavg" 2>&1 || true
  98                         ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true
  99                         ## /H*
 100
 101                         # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
 102                         # Otherwise exit, because we don't want HBase to do a
 103                         # full build.  We wouldn't normally do this check for smaller
 104                         # projects. :)
 105                         if [[ -z "${CHANGE_URL}" ]]; then
 106                             echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
 107                             exit 0
 108                         fi
 109                         # enable debug output for yetus
 110                         if [[ "true" = "${DEBUG}" ]]; then
 111                             YETUS_ARGS+=("--debug")
 112                         fi
 113                         YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
 114                         # where the source is located
 115                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 116                         # our project defaults come from a personality file
 117                         # which will get loaded automatically by setting the project name
 118                         YETUS_ARGS+=("--project=hbase")
 119                         # lots of different output formats
 120                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
 121                         YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
 122                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 123                         # enable writing back to Github
 124                         YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
 125                         YETUS_ARGS+=(--github-user=${GITHUB_USER})
 126                         # enable writing back to ASF JIRA
 127                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
 128                         YETUS_ARGS+=(--jira-user="${JIRA_USER}")
 129                         # auto-kill any surefire stragglers during unit test runs
 130                         YETUS_ARGS+=("--reapermode=kill")
 131                         YETUS_ARGS+=("--multijdktests=compile")
 132                         # set relatively high limits for ASF machines
 133                         # changing these to higher values may cause problems
 134                         # with other jobs on systemd-enabled machines
 135                         YETUS_ARGS+=("--proclimit=10000")
 136                         YETUS_ARGS+=("--dockermemlimit=20g")
 137                         # -1 findbugs issues that show up prior to the patch being applied
 138                         YETUS_ARGS+=("--findbugs-strict-precheck")
 139                         # rsync these files back into the archive dir
 140                         YETUS_ARGS+=("--archive-list=rat.txt")
 141                         # URL for user-side presentation in reports and such to our artifacts
 142                         # (needs to match the archive bits below)
 143                         YETUS_ARGS+=("--build-url-artifacts=artifact/out")
 144                         # plugins to enable
 145                         YETUS_ARGS+=("--plugins=all")
 146                         # don't let these tests cause -1s because we aren't really paying that
 147                         # much attention to them
 148                         YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests")
 149                         # run in docker mode and specifically point to our
 150                         # Dockerfile since we don't want to use the auto-pulled version.
 151                         YETUS_ARGS+=("--docker")
 152                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 153                         YETUS_ARGS+=("--mvn-custom-repos")
 154                         YETUS_ARGS+=("--jenkins")
 155                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64")
 156                         YETUS_ARGS+=("--findbugs-home=/usr")
 157                         YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
 158                         YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
 159                         YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 160                         YETUS_ARGS+=("--quick-hadoopcheck")
 161                         YETUS_ARGS+=("--skip-errorprone")
 162                         # effectively treat dev-support as a custom maven module
 163                         YETUS_ARGS+=("--skip-dir=dev-support")
 164                         # help keep the ASF boxes clean
 165                         YETUS_ARGS+=("--sentinel")
 166                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
 167                         '''
 168                 }
 169             }
 170         }
 171
 172     }
 173
 174     post {
 175         always {
 176           script {
 177             // Yetus output
 178             archiveArtifacts "${env.PATCHDIR}/**"
 179             // Publish the HTML report so that it can be looked at
 180             // Has to be relative to WORKSPACE.
 181             publishHTML (target: [
 182                           allowMissing: true,
 183                           keepAll: true,
 184                           alwaysLinkToLastBuild: true,
 185                           // Has to be relative to WORKSPACE
 186                           reportDir: "${env.PATCHDIR}",
 187                           reportFiles: 'report.html',
 188                           reportName: 'Yetus Report'
 189             ])
 190             // Publish JUnit results
 191             try {
 192                 junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
 193             } catch(e) {
 194                 echo 'junit processing: ' + e.toString()
 195             }
 196           }
 197         }
 198
 199         // Jenkins pipeline jobs fill slaves on PRs without this :(
 200         cleanup() {
 201             script {
 202                 sh '''
 203                     # See YETUS-764
 204                     if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
 205                       echo "test-patch process appears to still be running: killing"
 206                       kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
 207                       sleep 10
 208                     fi
 209                     if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
 210                       echo "test-patch container appears to still be running: killing"
 211                       docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
 212                     fi
 213                     # See HADOOP-13951
 214                     chmod -R u+rxw "${WORKSPACE}"
 215                     '''
 216                 deleteDir()
 217             }
 218         }
 219     }
 220 }