dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 10, unit: 'HOURS')
  29         timestamps()
  30         skipDefaultCheckout()
  31     }
  32
  33     environment {
  34         SRC_REL = 'src'
  35         PATCH_REL = 'output'
  36         YETUS_REL = 'yetus'
  37         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  38         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION = 'rel/0.12.0'
  41         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  42         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
  43         // output from surefire; sadly the archive function in yetus only works on file names.
  44         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  45         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
  46         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
  47         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  48
  49         // a global view of paths. parallel stages can land on the same host concurrently, so each
  50         // stage works in its own subdirectory. there is an "output" under each of these
  51         // directories, which we retrieve after the build is complete.
  52         WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
  53         WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
  54         WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  55         ASF_NIGHTLIES = 'https://nightlies.apache.org'
  56     }
  57
  58     parameters {
  59         booleanParam(name: 'DEBUG',
  60                defaultValue: false,
  61                description: 'Print extra outputs for debugging the jenkins job and yetus')
  62     }
  63
  64     stages {
  65         stage ('precommit checks') {
  66             parallel {
  67                 stage ('yetus general check') {
  68                     agent {
  69                         node {
  70                             label 'Hadoop'
  71                         }
  72                     }
  73                     environment {
  74                         // customized per parallel stage
  75                         PLUGINS = "${GENERAL_CHECK_PLUGINS}"
  76                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
  77                         WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  78                         // identical for all parallel stages
  79                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
  80                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
  81                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
  82                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
  83                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
  84                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
  85                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
  86                     }
  87                     steps {
  88                         dir("${SOURCEDIR}") {
  89                             checkout scm
  90                         }
  91                         dir("${YETUSDIR}") {
  92                             checkout([
  93                               $class           : 'GitSCM',
  94                               branches         : [[name: "${YETUS_VERSION}"]],
  95                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
  96                             )
  97                         }
  98                         dir("${WORKDIR}") {
  99                             withCredentials([
 100                                 usernamePassword(
 101                                   credentialsId: 'apache-hbase-at-github.com',
 102                                   passwordVariable: 'GITHUB_PASSWORD',
 103                                   usernameVariable: 'GITHUB_USER'
 104                                 )]) {
 105                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 106                                     hostname -a ; pwd ; ls -la
 107                                     printenv 2>&1 | sort
 108                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 109                                     "${YETUS_DRIVER}"
 110                                 '''
 111                             }
 112                         }
 113                     }
 114                     post {
 115                         always {
 116                             // Has to be relative to WORKSPACE.
 117                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 118                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 119                             publishHTML target: [
 120                               allowMissing: true,
 121                               keepAll: true,
 122                               alwaysLinkToLastBuild: true,
 123                               // Has to be relative to WORKSPACE
 124                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 125                               reportFiles: 'report.html',
 126                               reportName: 'PR General Check Report'
 127                             ]
 128                         }
 129                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 130                         cleanup() {
 131                             script {
 132                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 133                                     # See YETUS-764
 134                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 135                                       echo "test-patch process appears to still be running: killing"
 136                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 137                                       sleep 10
 138                                     fi
 139                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 140                                       echo "test-patch container appears to still be running: killing"
 141                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 142                                     fi
 143                                     # See HADOOP-13951
 144                                     chmod -R u+rxw "${WORKSPACE}"
 145                                 '''
 146                                 dir ("${WORKDIR}") {
 147                                     deleteDir()
 148                                 }
 149                             }
 150                         }
 151                     }
 152                 }
 153                 stage ('yetus jdk8 Hadoop3 checks') {
 154                     agent {
 155                         node {
 156                             label 'Hadoop'
 157                         }
 158                     }
 159                     environment {
 160                         // customized per parallel stage
 161                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 162                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
 163                         WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
 164                         // identical for all parallel stages
 165                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 166                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 167                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 168                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 169                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 170                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 171                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 172                         SKIP_ERRORPRONE = true
 173                     }
 174                     steps {
 175                         dir("${SOURCEDIR}") {
 176                             checkout scm
 177                         }
 178                         dir("${YETUSDIR}") {
 179                             checkout([
 180                               $class           : 'GitSCM',
 181                               branches         : [[name: "${YETUS_VERSION}"]],
 182                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 183                             )
 184                         }
 185                         dir("${WORKDIR}") {
 186                             withCredentials([
 187                               usernamePassword(
 188                                 credentialsId: 'apache-hbase-at-github.com',
 189                                 passwordVariable: 'GITHUB_PASSWORD',
 190                                 usernameVariable: 'GITHUB_USER'
 191                               )]) {
 192                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 193                                     hostname -a ; pwd ; ls -la
 194                                     printenv 2>&1 | sort
 195                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 196                                     "${YETUS_DRIVER}"
 197                                 '''
 198                             }
 199                         }
 200                     }
 201                     post {
 202                         always {
 203                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 204                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 205                                 if [ -d "${PATCHDIR}/archiver" ]; then
 206                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 207                                   if [[ 0 -ne ${count} ]]; then
 208                                     echo "zipping ${count} archived files"
 209                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 210                                   else
 211                                     echo "No archived files, skipping compressing."
 212                                   fi
 213                                 else
 214                                   echo "No archiver directory, skipping compressing."
 215                                 fi
 216                             '''
 217                             sshPublisher(publishers: [
 218                               sshPublisherDesc(configName: 'Nightlies',
 219                                 transfers: [
 220                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 221                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 222                                   )
 223                                 ]
 224                               )
 225                             ])
 226                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 227                             sh '''#!/bin/bash -e
 228                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 229                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 230                               rm -rf "${PATCHDIR}/test_logs.zip"
 231                               echo "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}/test_logs.zip" > "${PATCHDIR}/test_logs.txt"
 232                             else
 233                               echo "No test_logs.zip, skipping"
 234                             fi
 235                             '''
 236                             // Has to be relative to WORKSPACE.
 237                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 238                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 239                             publishHTML target: [
 240                               allowMissing: true,
 241                               keepAll: true,
 242                               alwaysLinkToLastBuild: true,
 243                               // Has to be relative to WORKSPACE
 244                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 245                               reportFiles: 'report.html',
 246                               reportName: 'PR JDK8 Hadoop3 Check Report'
 247                             ]
 248                         }
 249                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 250                         cleanup() {
 251                             script {
 252                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 253                                     # See YETUS-764
 254                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 255                                       echo "test-patch process appears to still be running: killing"
 256                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 257                                       sleep 10
 258                                     fi
 259                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 260                                       echo "test-patch container appears to still be running: killing"
 261                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 262                                     fi
 263                                     # See HADOOP-13951
 264                                     chmod -R u+rxw "${WORKSPACE}"
 265                                 '''
 266                                 dir ("${WORKDIR}") {
 267                                     deleteDir()
 268                                 }
 269                             }
 270                         }
 271                     }
 272                 }
 273                 stage ('yetus jdk11 hadoop3 checks') {
 274                     agent {
 275                         node {
 276                             label 'Hadoop'
 277                         }
 278                     }
 279                     environment {
 280                         // customized per parallel stage
 281                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 282                         SET_JAVA_HOME = '/usr/lib/jvm/java-11'
 283                         WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
 284                         // identical for all parallel stages
 285                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 286                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 287                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 288                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 289                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 290                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 291                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 292                         SKIP_ERRORPRONE = true
 293                     }
 294                     steps {
 295                         dir("${SOURCEDIR}") {
 296                             checkout scm
 297                         }
 298                         dir("${YETUSDIR}") {
 299                             checkout([
 300                               $class           : 'GitSCM',
 301                               branches         : [[name: "${YETUS_VERSION}"]],
 302                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 303                             )
 304                         }
 305                         dir("${WORKDIR}") {
 306                             withCredentials([
 307                               usernamePassword(
 308                                 credentialsId: 'apache-hbase-at-github.com',
 309                                 passwordVariable: 'GITHUB_PASSWORD',
 310                                 usernameVariable: 'GITHUB_USER'
 311                               )]) {
 312                                 sh label: 'test-patch', script: '''#!/bin/bash -e
 313                                     hostname -a ; pwd ; ls -la
 314                                     printenv 2>&1 | sort
 315                                     echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 316                                     "${YETUS_DRIVER}"
 317                                 '''
 318                             }
 319                         }
 320                     }
 321                     post {
 322                         always {
 323                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 324                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 325                                 if [ -d "${PATCHDIR}/archiver" ]; then
 326                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 327                                   if [[ 0 -ne ${count} ]]; then
 328                                     echo "zipping ${count} archived files"
 329                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 330                                   else
 331                                     echo "No archived files, skipping compressing."
 332                                   fi
 333                                 else
 334                                   echo "No archiver directory, skipping compressing."
 335                                 fi
 336                             '''
 337                             sshPublisher(publishers: [
 338                               sshPublisherDesc(configName: 'Nightlies',
 339                                 transfers: [
 340                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 341                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 342                                   )
 343                                 ]
 344                               )
 345                             ])
 346                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 347                             sh '''#!/bin/bash -e
 348                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 349                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 350                               rm -rf "${PATCHDIR}/test_logs.zip"
 351                               echo "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}/test_logs.zip" > "${PATCHDIR}/test_logs.txt"
 352                             else
 353                               echo "No test_logs.zip, skipping"
 354                             fi
 355                             '''
 356                             // Has to be relative to WORKSPACE.
 357                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 358                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 359                             publishHTML target: [
 360                               allowMissing: true,
 361                               keepAll: true,
 362                               alwaysLinkToLastBuild: true,
 363                               // Has to be relative to WORKSPACE
 364                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 365                               reportFiles: 'report.html',
 366                               reportName: 'PR JDK11 Hadoop3 Check Report'
 367                             ]
 368                         }
 369                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 370                         cleanup() {
 371                             script {
 372                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 373                                     # See YETUS-764
 374                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 375                                       echo "test-patch process appears to still be running: killing"
 376                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 377                                       sleep 10
 378                                     fi
 379                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 380                                       echo "test-patch container appears to still be running: killing"
 381                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 382                                     fi
 383                                     # See HADOOP-13951
 384                                     chmod -R u+rxw "${WORKSPACE}"
 385                                 '''
 386                                 dir ("${WORKDIR}") {
 387                                     deleteDir()
 388                                 }
 389                             }
 390                         }
 391                     }
 392                 }
 393             }
 394         }
 395     }
 396
 397     post {
 398         // Jenkins pipeline jobs fill slaves on PRs without this :(
 399         cleanup() {
 400             script {
 401                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 402                     # See HADOOP-13951
 403                     chmod -R u+rxw "${WORKSPACE}"
 404                     '''
 405                 deleteDir()
 406             }
 407         }
 408     }
 409 }