dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'Hadoop'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 10, unit: 'HOURS')
  29         timestamps()
  30         skipDefaultCheckout()
  31     }
  32
  33     environment {
  34         SRC_REL = 'src'
  35         PATCH_REL = 'output'
  36         YETUS_REL = 'yetus'
  37         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  38         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION = 'rel/0.12.0'
  41         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  42         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
  43         // output from surefire; sadly the archive function in yetus only works on file names.
  44         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  45         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
  46         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
  47         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  48
  49         // a global view of paths. parallel stages can land on the same host concurrently, so each
  50         // stage works in its own subdirectory. there is an "output" under each of these
  51         // directories, which we retrieve after the build is complete.
  52         WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
  53         WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
  54         WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  55         ASF_NIGHTLIES = 'https://nightlies.apache.org'
  56     }
  57
  58     parameters {
  59         booleanParam(name: 'DEBUG',
  60                defaultValue: false,
  61                description: 'Print extra outputs for debugging the jenkins job and yetus')
  62     }
  63
  64     stages {
  65         stage ('precommit checks') {
  66             parallel {
  67                 stage ('yetus general check') {
  68                     agent {
  69                         node {
  70                             label 'Hadoop'
  71                         }
  72                     }
  73                     environment {
  74                         // customized per parallel stage
  75                         PLUGINS = "${GENERAL_CHECK_PLUGINS}"
  76                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
  77                         WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  78                         // identical for all parallel stages
  79                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
  80                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
  81                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
  82                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
  83                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
  84                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
  85                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
  86                     }
  87                     steps {
  88                         dir("${SOURCEDIR}") {
  89                             checkout scm
  90                         }
  91                         dir("${YETUSDIR}") {
  92                             checkout([
  93                               $class           : 'GitSCM',
  94                               branches         : [[name: "${YETUS_VERSION}"]],
  95                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
  96                             )
  97                         }
  98                         dir("${WORKDIR}") {
  99                             withCredentials([
 100                                 usernamePassword(
 101                                   credentialsId: 'apache-hbase-at-github.com',
 102                                   passwordVariable: 'GITHUB_PASSWORD',
 103                                   usernameVariable: 'GITHUB_USER'
 104                                 )]) {
 105                                 script {
 106                                   def ret = sh(
 107                                     label: 'test-patch',
 108                                     returnStatus: true,
 109                                     script: '''#!/bin/bash -e
 110                                       hostname -a ; pwd ; ls -la
 111                                       printenv 2>&1 | sort
 112                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 113                                       "${YETUS_DRIVER}"
 114                                     '''
 115                                   )
 116                                   if (ret != 0) {
 117                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 118                                     // test output. See HBASE-26339 for more details.
 119                                     currentBuild.result = 'UNSTABLE'
 120                                   }
 121                                 }
 122                             }
 123                         }
 124                     }
 125                     post {
 126                         always {
 127                             // Has to be relative to WORKSPACE.
 128                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 129                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 130                             publishHTML target: [
 131                               allowMissing: true,
 132                               keepAll: true,
 133                               alwaysLinkToLastBuild: true,
 134                               // Has to be relative to WORKSPACE
 135                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 136                               reportFiles: 'report.html',
 137                               reportName: 'PR General Check Report'
 138                             ]
 139                         }
 140                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 141                         cleanup() {
 142                             script {
 143                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 144                                     # See YETUS-764
 145                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 146                                       echo "test-patch process appears to still be running: killing"
 147                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 148                                       sleep 10
 149                                     fi
 150                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 151                                       echo "test-patch container appears to still be running: killing"
 152                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 153                                     fi
 154                                     # See HADOOP-13951
 155                                     chmod -R u+rxw "${WORKSPACE}"
 156                                 '''
 157                                 dir ("${WORKDIR}") {
 158                                     deleteDir()
 159                                 }
 160                             }
 161                         }
 162                     }
 163                 }
 164                 stage ('yetus jdk8 Hadoop3 checks') {
 165                     agent {
 166                         node {
 167                             label 'Hadoop'
 168                         }
 169                     }
 170                     environment {
 171                         // customized per parallel stage
 172                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 173                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
 174                         WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
 175                         // identical for all parallel stages
 176                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 177                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 178                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 179                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 180                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 181                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 182                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 183                         SKIP_ERRORPRONE = true
 184                     }
 185                     steps {
 186                         dir("${SOURCEDIR}") {
 187                             checkout scm
 188                         }
 189                         dir("${YETUSDIR}") {
 190                             checkout([
 191                               $class           : 'GitSCM',
 192                               branches         : [[name: "${YETUS_VERSION}"]],
 193                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 194                             )
 195                         }
 196                         dir("${WORKDIR}") {
 197                             withCredentials([
 198                               usernamePassword(
 199                                 credentialsId: 'apache-hbase-at-github.com',
 200                                 passwordVariable: 'GITHUB_PASSWORD',
 201                                 usernameVariable: 'GITHUB_USER'
 202                               )]) {
 203                                 script {
 204                                   def ret = sh(
 205                                     label: 'test-patch',
 206                                     returnStatus: true,
 207                                     script: '''#!/bin/bash -e
 208                                       hostname -a ; pwd ; ls -la
 209                                       printenv 2>&1 | sort
 210                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 211                                       "${YETUS_DRIVER}"
 212                                     '''
 213                                   )
 214                                   if (ret != 0) {
 215                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 216                                     // test output. See HBASE-26339 for more details.
 217                                     currentBuild.result = 'UNSTABLE'
 218                                   }
 219                                 }
 220                             }
 221                         }
 222                     }
 223                     post {
 224                         always {
 225                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 226                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 227                                 if [ -d "${PATCHDIR}/archiver" ]; then
 228                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 229                                   if [[ 0 -ne ${count} ]]; then
 230                                     echo "zipping ${count} archived files"
 231                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 232                                   else
 233                                     echo "No archived files, skipping compressing."
 234                                   fi
 235                                 else
 236                                   echo "No archiver directory, skipping compressing."
 237                                 fi
 238                             '''
 239                             sshPublisher(publishers: [
 240                               sshPublisherDesc(configName: 'Nightlies',
 241                                 transfers: [
 242                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 243                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 244                                   )
 245                                 ]
 246                               )
 247                             ])
 248                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 249                             sh '''#!/bin/bash -e
 250                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 251                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 252                               rm -rf "${PATCHDIR}/test_logs.zip"
 253                               echo "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}/test_logs.zip" > "${PATCHDIR}/test_logs.txt"
 254                             else
 255                               echo "No test_logs.zip, skipping"
 256                             fi
 257                             '''
 258                             // Has to be relative to WORKSPACE.
 259                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 260                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 261                             publishHTML target: [
 262                               allowMissing: true,
 263                               keepAll: true,
 264                               alwaysLinkToLastBuild: true,
 265                               // Has to be relative to WORKSPACE
 266                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 267                               reportFiles: 'report.html',
 268                               reportName: 'PR JDK8 Hadoop3 Check Report'
 269                             ]
 270                         }
 271                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 272                         cleanup() {
 273                             script {
 274                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 275                                     # See YETUS-764
 276                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 277                                       echo "test-patch process appears to still be running: killing"
 278                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 279                                       sleep 10
 280                                     fi
 281                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 282                                       echo "test-patch container appears to still be running: killing"
 283                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 284                                     fi
 285                                     # See HADOOP-13951
 286                                     chmod -R u+rxw "${WORKSPACE}"
 287                                 '''
 288                                 dir ("${WORKDIR}") {
 289                                     deleteDir()
 290                                 }
 291                             }
 292                         }
 293                     }
 294                 }
 295                 stage ('yetus jdk11 hadoop3 checks') {
 296                     agent {
 297                         node {
 298                             label 'Hadoop'
 299                         }
 300                     }
 301                     environment {
 302                         // customized per parallel stage
 303                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 304                         SET_JAVA_HOME = '/usr/lib/jvm/java-11'
 305                         WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
 306                         // identical for all parallel stages
 307                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 308                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 309                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 310                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 311                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 312                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 313                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 314                         SKIP_ERRORPRONE = true
 315                     }
 316                     steps {
 317                         dir("${SOURCEDIR}") {
 318                             checkout scm
 319                         }
 320                         dir("${YETUSDIR}") {
 321                             checkout([
 322                               $class           : 'GitSCM',
 323                               branches         : [[name: "${YETUS_VERSION}"]],
 324                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 325                             )
 326                         }
 327                         dir("${WORKDIR}") {
 328                             withCredentials([
 329                               usernamePassword(
 330                                 credentialsId: 'apache-hbase-at-github.com',
 331                                 passwordVariable: 'GITHUB_PASSWORD',
 332                                 usernameVariable: 'GITHUB_USER'
 333                               )]) {
 334                                 script {
 335                                   def ret = sh(
 336                                     label: 'test-patch',
 337                                     returnStatus: true,
 338                                     script: '''#!/bin/bash -e
 339                                       hostname -a ; pwd ; ls -la
 340                                       printenv 2>&1 | sort
 341                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 342                                       "${YETUS_DRIVER}"
 343                                     '''
 344                                   )
 345                                   if (ret != 0) {
 346                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 347                                     // test output. See HBASE-26339 for more details.
 348                                     currentBuild.result = 'UNSTABLE'
 349                                   }
 350                                 }
 351                             }
 352                         }
 353                     }
 354                     post {
 355                         always {
 356                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", allowEmptyResults: true
 357                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 358                                 if [ -d "${PATCHDIR}/archiver" ]; then
 359                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 360                                   if [[ 0 -ne ${count} ]]; then
 361                                     echo "zipping ${count} archived files"
 362                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 363                                   else
 364                                     echo "No archived files, skipping compressing."
 365                                   fi
 366                                 else
 367                                   echo "No archiver directory, skipping compressing."
 368                                 fi
 369                             '''
 370                             sshPublisher(publishers: [
 371                               sshPublisherDesc(configName: 'Nightlies',
 372                                 transfers: [
 373                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 374                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 375                                   )
 376                                 ]
 377                               )
 378                             ])
 379                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 380                             sh '''#!/bin/bash -e
 381                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 382                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 383                               rm -rf "${PATCHDIR}/test_logs.zip"
 384                               echo "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}/test_logs.zip" > "${PATCHDIR}/test_logs.txt"
 385                             else
 386                               echo "No test_logs.zip, skipping"
 387                             fi
 388                             '''
 389                             // Has to be relative to WORKSPACE.
 390                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 391                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 392                             publishHTML target: [
 393                               allowMissing: true,
 394                               keepAll: true,
 395                               alwaysLinkToLastBuild: true,
 396                               // Has to be relative to WORKSPACE
 397                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 398                               reportFiles: 'report.html',
 399                               reportName: 'PR JDK11 Hadoop3 Check Report'
 400                             ]
 401                         }
 402                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 403                         cleanup() {
 404                             script {
 405                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 406                                     # See YETUS-764
 407                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 408                                       echo "test-patch process appears to still be running: killing"
 409                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 410                                       sleep 10
 411                                     fi
 412                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 413                                       echo "test-patch container appears to still be running: killing"
 414                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 415                                     fi
 416                                     # See HADOOP-13951
 417                                     chmod -R u+rxw "${WORKSPACE}"
 418                                 '''
 419                                 dir ("${WORKDIR}") {
 420                                     deleteDir()
 421                                 }
 422                             }
 423                         }
 424                     }
 425                 }
 426             }
 427         }
 428     }
 429
 430     post {
 431         // Jenkins pipeline jobs fill slaves on PRs without this :(
 432         cleanup() {
 433             script {
 434                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 435                     # See HADOOP-13951
 436                     chmod -R u+rxw "${WORKSPACE}"
 437                     '''
 438                 deleteDir()
 439             }
 440         }
 441     }
 442 }