dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 10, unit: 'HOURS')
  29         timestamps()
  30         skipDefaultCheckout()
  31     }
  32
  33     environment {
  34         SRC_REL = 'src'
  35         PATCH_REL = 'output'
  36         YETUS_REL = 'yetus'
  37         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  38         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION = 'rel/0.12.0'
  41         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  42         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
  43         // output from surefire; sadly the archive function in yetus only works on file names.
  44         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  45         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
  46         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
  47         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  48
  49         // a global view of paths. parallel stages can land on the same host concurrently, so each
  50         // stage works in its own subdirectory. there is an "output" under each of these
  51         // directories, which we retrieve after the build is complete.
  52         WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
  53         WORKDIR_REL_JDK8_HADOOP2_CHECK = 'yetus-jdk8-hadoop2-check'
  54         WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  55     }
  56
  57     parameters {
  58         booleanParam(name: 'DEBUG',
  59                defaultValue: false,
  60                description: 'Print extra outputs for debugging the jenkins job and yetus')
  61     }
  62
  63     stages {
  64         stage ('precommit checks') {
  65             parallel {
  66                 stage ('yetus general check') {
  67                     agent {
  68                         node {
  69                             label 'Hadoop'
  70                         }
  71                     }
  72                     environment {
  73                         // customized per parallel stage
  74                         PLUGINS = "${GENERAL_CHECK_PLUGINS}"
  75                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
  76                         WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  77                         // identical for all parallel stages
  78                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
  79                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
  80                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
  81                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
  82                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
  83                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
  84                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
  85                     }
  86                     steps {
  87                         dir("${SOURCEDIR}") {
  88                             checkout scm
  89                         }
  90                         dir("${YETUSDIR}") {
  91                             checkout([
  92                               $class           : 'GitSCM',
  93                               branches         : [[name: "${YETUS_VERSION}"]],
  94                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
  95                             )
  96                         }
  97                         dir("${WORKDIR}") {
  98                             withCredentials([
  99                                 usernamePassword(
 100                                   credentialsId: 'apache-hbase-at-github.com',
 101                                   passwordVariable: 'GITHUB_PASSWORD',
 102                                   usernameVariable: 'GITHUB_USER'
 103                                 )]) {
 104                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 105                                     hostname -a ; pwd ; ls -la
 106                                     printenv 2>&1 | sort
 107                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 108                                     "${YETUS_DRIVER}"
 109                                 '''
 110                             }
 111                         }
 112                     }
 113                     post {
 114                         always {
 115                             // Has to be relative to WORKSPACE.
 116                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 117                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 118                             publishHTML target: [
 119                               allowMissing: true,
 120                               keepAll: true,
 121                               alwaysLinkToLastBuild: true,
 122                               // Has to be relative to WORKSPACE
 123                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 124                               reportFiles: 'report.html',
 125                               reportName: 'PR General Check Report'
 126                             ]
 127                         }
 128                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 129                         cleanup() {
 130                             script {
 131                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 132                                     # See YETUS-764
 133                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 134                                       echo "test-patch process appears to still be running: killing"
 135                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 136                                       sleep 10
 137                                     fi
 138                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 139                                       echo "test-patch container appears to still be running: killing"
 140                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 141                                     fi
 142                                     # See HADOOP-13951
 143                                     chmod -R u+rxw "${WORKSPACE}"
 144                                 '''
 145                                 dir ("${WORKDIR}") {
 146                                     deleteDir()
 147                                 }
 148                             }
 149                         }
 150                     }
 151                 }
 152                 stage ('yetus jdk8 Hadoop2 checks') {
 153                     agent {
 154                         node {
 155                             label 'Hadoop'
 156                         }
 157                     }
 158                     environment {
 159                         // customized per parallel stage
 160                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 161                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
 162                         WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP2_CHECK}"
 163                         // identical for all parallel stages
 164                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 165                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 166                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 167                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 168                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 169                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 170                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 171                         SKIP_ERRORPRONE = true
 172                     }
 173                     steps {
 174                         dir("${SOURCEDIR}") {
 175                             checkout scm
 176                         }
 177                         dir("${YETUSDIR}") {
 178                             checkout([
 179                               $class           : 'GitSCM',
 180                               branches         : [[name: "${YETUS_VERSION}"]],
 181                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 182                             )
 183                         }
 184                         dir("${WORKDIR}") {
 185                             withCredentials([
 186                               usernamePassword(
 187                                 credentialsId: 'apache-hbase-at-github.com',
 188                                 passwordVariable: 'GITHUB_PASSWORD',
 189                                 usernameVariable: 'GITHUB_USER'
 190                               )]) {
 191                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 192                                     hostname -a ; pwd ; ls -la
 193                                     printenv 2>&1 | sort
 194                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 195                                     "${YETUS_DRIVER}"
 196                                 '''
 197                             }
 198                         }
 199                     }
 200                     post {
 201                         always {
 202                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 203                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 204                                 if [ -d "${PATCHDIR}/archiver" ]; then
 205                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 206                                   if [[ 0 -ne ${count} ]]; then
 207                                     echo "zipping ${count} archived files"
 208                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 209                                   else
 210                                     echo "No archived files, skipping compressing."
 211                                   fi
 212                                 else
 213                                   echo "No archiver directory, skipping compressing."
 214                                 fi
 215                             '''
 216                             // Has to be relative to WORKSPACE.
 217                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 218                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 219                             publishHTML target: [
 220                               allowMissing: true,
 221                               keepAll: true,
 222                               alwaysLinkToLastBuild: true,
 223                               // Has to be relative to WORKSPACE
 224                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 225                               reportFiles: 'report.html',
 226                               reportName: 'PR JDK8 Hadoop3 Check Report'
 227                             ]
 228                         }
 229                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 230                         cleanup() {
 231                             script {
 232                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 233                                     # See YETUS-764
 234                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 235                                       echo "test-patch process appears to still be running: killing"
 236                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 237                                       sleep 10
 238                                     fi
 239                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 240                                       echo "test-patch container appears to still be running: killing"
 241                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 242                                     fi
 243                                     # See HADOOP-13951
 244                                     chmod -R u+rxw "${WORKSPACE}"
 245                                 '''
 246                                 dir ("${WORKDIR}") {
 247                                     deleteDir()
 248                                 }
 249                             }
 250                         }
 251                     }
 252                 }
 253                 stage ('yetus jdk11 hadoop3 checks') {
 254                     agent {
 255                         node {
 256                             label 'Hadoop'
 257                         }
 258                     }
 259                     environment {
 260                         // customized per parallel stage
 261                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 262                         SET_JAVA_HOME = '/usr/lib/jvm/java-11'
 263                         HADOOP_PROFILE = '3.0'
 264                         WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
 265                         // identical for all parallel stages
 266                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 267                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 268                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 269                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 270                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 271                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 272                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 273                         SKIP_ERRORPRONE = true
 274                     }
 275                     steps {
 276                         dir("${SOURCEDIR}") {
 277                             checkout scm
 278                         }
 279                         dir("${YETUSDIR}") {
 280                             checkout([
 281                               $class           : 'GitSCM',
 282                               branches         : [[name: "${YETUS_VERSION}"]],
 283                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 284                             )
 285                         }
 286                         dir("${WORKDIR}") {
 287                             withCredentials([
 288                               usernamePassword(
 289                                 credentialsId: 'apache-hbase-at-github.com',
 290                                 passwordVariable: 'GITHUB_PASSWORD',
 291                                 usernameVariable: 'GITHUB_USER'
 292                               )]) {
 293                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 294                                     hostname -a ; pwd ; ls -la
 295                                     printenv 2>&1 | sort
 296                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 297                                     "${YETUS_DRIVER}"
 298                                 '''
 299                             }
 300                         }
 301                     }
 302                     post {
 303                         always {
 304                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 305                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 306                                 if [ -d "${PATCHDIR}/archiver" ]; then
 307                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 308                                   if [[ 0 -ne ${count} ]]; then
 309                                     echo "zipping ${count} archived files"
 310                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 311                                   else
 312                                     echo "No archived files, skipping compressing."
 313                                   fi
 314                                 else
 315                                   echo "No archiver directory, skipping compressing."
 316                                 fi
 317                             '''
 318                             // Has to be relative to WORKSPACE.
 319                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 320                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 321                             publishHTML target: [
 322                               allowMissing: true,
 323                               keepAll: true,
 324                               alwaysLinkToLastBuild: true,
 325                               // Has to be relative to WORKSPACE
 326                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 327                               reportFiles: 'report.html',
 328                               reportName: 'PR JDK11 Hadoop3 Check Report'
 329                             ]
 330                         }
 331                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 332                         cleanup() {
 333                             script {
 334                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 335                                     # See YETUS-764
 336                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 337                                       echo "test-patch process appears to still be running: killing"
 338                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 339                                       sleep 10
 340                                     fi
 341                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 342                                       echo "test-patch container appears to still be running: killing"
 343                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 344                                     fi
 345                                     # See HADOOP-13951
 346                                     chmod -R u+rxw "${WORKSPACE}"
 347                                 '''
 348                                 dir ("${WORKDIR}") {
 349                                     deleteDir()
 350                                 }
 351                             }
 352                         }
 353                     }
 354                 }
 355             }
 356         }
 357     }
 358
 359     post {
 360         // Jenkins pipeline jobs fill slaves on PRs without this :(
 361         cleanup() {
 362             script {
 363                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 364                     # See HADOOP-13951
 365                     chmod -R u+rxw "${WORKSPACE}"
 366                     '''
 367                 deleteDir()
 368             }
 369         }
 370     }
 371 }