dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 7, unit: 'HOURS')
  29         timestamps()
  30         checkoutToSubdirectory('src')
  31     }
  32
  33     environment {
  34         SOURCEDIR = 'src'
  35         // will also need to change notification section below
  36         PATCHDIR = 'out'
  37         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
  38         YETUS='yetus'
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION='rel/0.11.1'
  41     }
  42
  43     parameters {
  44         booleanParam(name: 'DEBUG',
  45                defaultValue: false,
  46                description: 'Print extra outputs for debugging the jenkins job and yetus')
  47     }
  48
  49     stages {
  50         stage ('install yetus') {
  51             steps {
  52                 dir("${WORKSPACE}/${YETUS}") {
  53                     checkout([
  54                         $class: 'GitSCM',
  55                         branches: [[name: "${env.YETUS_VERSION}"]],
  56                         userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
  57                     )
  58                 }
  59             }
  60         }
  61
  62         stage ('precommit-run') {
  63             steps {
  64                 withCredentials(
  65                     [usernamePassword(credentialsId: 'apache-hbase-at-github.com',
  66                                   passwordVariable: 'GITHUB_PASSWORD',
  67                                   usernameVariable: 'GITHUB_USER'),
  68                     usernamePassword(credentialsId: 'hbaseqa-at-asf-jira',
  69                                         passwordVariable: 'JIRA_PASSWORD',
  70                                         usernameVariable: 'JIRA_USER')]) {
  71                         sh '''#!/usr/bin/env bash
  72                         set -e
  73                         TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
  74                         # this must be clean for every run
  75                         if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
  76                           rm -rf "${WORKSPACE}/${PATCHDIR}"
  77                         fi
  78                         mkdir -p "${WORKSPACE}/${PATCHDIR}"
  79
  80                         ## Checking on H* machine nonsense
  81                         echo "JAVA_HOME: ${JAVA_HOME}"
  82                         ls -l "${JAVA_HOME}" || true
  83                         echo "MAVEN_HOME: ${MAVEN_HOME}"
  84                         echo "maven version:"
  85                         mvn --offline --version  || true
  86                         mkdir "${PATCHDIR}/machine"
  87                        "${SOURCEDIR}/dev-support/gather_machine_environment.sh" "${PATCHDIR}/machine"
  88                         ## /H*
  89
  90                         # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
  91                         # Otherwise exit, because we don't want HBase to do a
  92                         # full build.  We wouldn't normally do this check for smaller
  93                         # projects. :)
  94                         if [[ -z "${CHANGE_URL}" ]]; then
  95                             echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
  96                             exit 0
  97                         fi
  98                         # enable debug output for yetus
  99                         if [[ "true" = "${DEBUG}" ]]; then
 100                             YETUS_ARGS+=("--debug")
 101                         fi
 102                         # If we're doing docker, make sure we don't accidentally pollute the image with a host java path
 103                         if [ -n "${JAVA_HOME}" ]; then
 104                           unset JAVA_HOME
 105                         fi
 106                         YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
 107                         # where the source is located
 108                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 109                         # our project defaults come from a personality file
 110                         # which will get loaded automatically by setting the project name
 111                         YETUS_ARGS+=("--project=hbase")
 112                         # lots of different output formats
 113                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
 114                         YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
 115                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 116                         # enable writing back to Github
 117                         YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
 118                         YETUS_ARGS+=(--github-user=${GITHUB_USER})
 119                         # enable writing back to ASF JIRA
 120                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
 121                         YETUS_ARGS+=(--jira-user="${JIRA_USER}")
 122                         # auto-kill any surefire stragglers during unit test runs
 123                         YETUS_ARGS+=("--reapermode=kill")
 124                         YETUS_ARGS+=("--multijdktests=compile")
 125                         # set relatively high limits for ASF machines
 126                         # changing these to higher values may cause problems
 127                         # with other jobs on systemd-enabled machines
 128                         YETUS_ARGS+=("--proclimit=10000")
 129                         YETUS_ARGS+=("--dockermemlimit=20g")
 130                         # -1 findbugs issues that show up prior to the patch being applied
 131                         YETUS_ARGS+=("--findbugs-strict-precheck")
 132                         # rsync these files back into the archive dir
 133                         YETUS_ARGS+=("--archive-list=rat.txt")
 134                         # URL for user-side presentation in reports and such to our artifacts
 135                         # (needs to match the archive bits below)
 136                         YETUS_ARGS+=("--build-url-artifacts=artifact/out")
 137                         # plugins to enable
 138                         YETUS_ARGS+=("--plugins=all")
 139                         # don't let these tests cause -1s because we aren't really paying that
 140                         # much attention to them
 141                         YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests")
 142                         # run in docker mode and specifically point to our
 143                         # Dockerfile since we don't want to use the auto-pulled version.
 144                         YETUS_ARGS+=("--docker")
 145                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 146                         YETUS_ARGS+=("--mvn-custom-repos")
 147                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64")
 148                         YETUS_ARGS+=("--findbugs-home=/usr")
 149                         YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
 150                         YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
 151                         YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 152                         YETUS_ARGS+=("--quick-hadoopcheck")
 153                         YETUS_ARGS+=("--skip-errorprone")
 154                         # effectively treat dev-support as a custom maven module
 155                         YETUS_ARGS+=("--skip-dirs=dev-support")
 156                         # help keep the ASF boxes clean
 157                         YETUS_ARGS+=("--sentinel")
 158                         # use emoji vote so it is easier to find the broken line
 159                         YETUS_ARGS+=("--github-use-emoji-vote")
 160                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
 161                         '''
 162                 }
 163             }
 164         }
 165
 166     }
 167
 168     post {
 169         always {
 170           script {
 171             // Yetus output
 172             archiveArtifacts "${env.PATCHDIR}/**"
 173             // Publish the HTML report so that it can be looked at
 174             // Has to be relative to WORKSPACE.
 175             publishHTML (target: [
 176                           allowMissing: true,
 177                           keepAll: true,
 178                           alwaysLinkToLastBuild: true,
 179                           // Has to be relative to WORKSPACE
 180                           reportDir: "${env.PATCHDIR}",
 181                           reportFiles: 'report.html',
 182                           reportName: 'Yetus Report'
 183             ])
 184             // Publish JUnit results
 185             try {
 186                 junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
 187             } catch(e) {
 188                 echo 'junit processing: ' + e.toString()
 189             }
 190           }
 191         }
 192
 193         // Jenkins pipeline jobs fill slaves on PRs without this :(
 194         cleanup() {
 195             script {
 196                 sh '''
 197                     # See YETUS-764
 198                     if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
 199                       echo "test-patch process appears to still be running: killing"
 200                       kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
 201                       sleep 10
 202                     fi
 203                     if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
 204                       echo "test-patch container appears to still be running: killing"
 205                       docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
 206                     fi
 207                     # See HADOOP-13951
 208                     chmod -R u+rxw "${WORKSPACE}"
 209                     '''
 210                 deleteDir()
 211             }
 212         }
 213     }
 214 }