dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'hbase'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 10, unit: 'HOURS')
  29         timestamps()
  30         skipDefaultCheckout()
  31     }
  32
  33     environment {
  34         SRC_REL = 'src'
  35         PATCH_REL = 'output'
  36         YETUS_REL = 'yetus'
  37         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  38         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION = 'rel/0.12.0'
  41         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  42         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
  43         // output from surefire; sadly the archive function in yetus only works on file names.
  44         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  45         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
  46         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
  47         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  48         // set build parallel
  49         BUILD_THREAD = 4
  50         SUREFIRE_FIRST_PART_FORK_COUNT = '0.5C'
  51         SUREFIRE_SECOND_PART_FORK_COUNT = '0.5C'
  52         // a global view of paths. parallel stages can land on the same host concurrently, so each
  53         // stage works in its own subdirectory. there is an "output" under each of these
  54         // directories, which we retrieve after the build is complete.
  55         WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
  56         WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
  57         WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  58         ASF_NIGHTLIES = 'https://nightlies.apache.org'
  59         ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
  60         ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
  61     }
  62
  63     parameters {
  64         booleanParam(name: 'DEBUG',
  65                defaultValue: false,
  66                description: 'Print extra outputs for debugging the jenkins job and yetus')
  67     }
  68
  69     stages {
  70         stage ('precommit checks') {
  71             parallel {
  72                 stage ('yetus general check') {
  73                     agent {
  74                         node {
  75                             label 'hbase'
  76                         }
  77                     }
  78                     environment {
  79                         // customized per parallel stage
  80                         PLUGINS = "${GENERAL_CHECK_PLUGINS}"
  81                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
  82                         WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  83                         // identical for all parallel stages
  84                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
  85                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
  86                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
  87                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
  88                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
  89                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
  90                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
  91                         ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}"
  92                     }
  93                     steps {
  94                         dir("${SOURCEDIR}") {
  95                             checkout scm
  96                         }
  97                         dir("${YETUSDIR}") {
  98                             checkout([
  99                               $class           : 'GitSCM',
 100                               branches         : [[name: "${YETUS_VERSION}"]],
 101                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 102                             )
 103                         }
 104                         dir("${WORKDIR}") {
 105                             withCredentials([
 106                                 usernamePassword(
 107                                   credentialsId: 'apache-hbase-at-github.com',
 108                                   passwordVariable: 'GITHUB_PASSWORD',
 109                                   usernameVariable: 'GITHUB_USER'
 110                                 )]) {
 111                                 script {
 112                                   def ret = sh(
 113                                     label: 'test-patch',
 114                                     returnStatus: true,
 115                                     script: '''#!/bin/bash -e
 116                                       hostname -a ; pwd ; ls -la
 117                                       printenv 2>&1 | sort
 118                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 119                                       "${YETUS_DRIVER}"
 120                                     '''
 121                                   )
 122                                   if (ret != 0) {
 123                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 124                                     // test output. See HBASE-26339 for more details.
 125                                     currentBuild.result = 'UNSTABLE'
 126                                   }
 127                                 }
 128                             }
 129                         }
 130                     }
 131                     post {
 132                         always {
 133                             sshPublisher(publishers: [
 134                               sshPublisherDesc(configName: 'Nightlies',
 135                                 transfers: [
 136                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 137                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*"
 138                                   )
 139                                 ]
 140                               )
 141                             ])
 142                             sh '''#!/bin/bash -e
 143                             if [ -d "${PATCHDIR}/branch-site" ]; then
 144                               echo "Remove ${PATCHDIR}/branch-site for saving space"
 145                               rm -rf "${PATCHDIR}/branch-site"
 146                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
 147                             else
 148                               echo "No branch-site, skipping"
 149                             fi
 150                             if [ -d "${PATCHDIR}/patch-site" ]; then
 151                               echo "Remove ${PATCHDIR}/patch-site for saving space"
 152                               rm -rf "${PATCHDIR}/patch-site"
 153                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
 154                             else
 155                               echo "No patch-site, skipping"
 156                             fi
 157                             '''
 158                             // Has to be relative to WORKSPACE.
 159                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*"
 160                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*"
 161                             publishHTML target: [
 162                               allowMissing: true,
 163                               keepAll: true,
 164                               alwaysLinkToLastBuild: true,
 165                               // Has to be relative to WORKSPACE
 166                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 167                               reportFiles: 'report.html',
 168                               reportName: 'PR General Check Report'
 169                             ]
 170                         }
 171                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 172                         cleanup() {
 173                             script {
 174                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 175                                     # See YETUS-764
 176                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 177                                       echo "test-patch process appears to still be running: killing"
 178                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 179                                       sleep 10
 180                                     fi
 181                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 182                                       echo "test-patch container appears to still be running: killing"
 183                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 184                                     fi
 185                                     # See HADOOP-13951
 186                                     chmod -R u+rxw "${WORKSPACE}"
 187                                 '''
 188                                 dir ("${WORKDIR}") {
 189                                     deleteDir()
 190                                 }
 191                             }
 192                         }
 193                     }
 194                 }
 195                 stage ('yetus jdk8 Hadoop3 checks') {
 196                     agent {
 197                         node {
 198                             label 'hbase'
 199                         }
 200                     }
 201                     environment {
 202                         // customized per parallel stage
 203                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 204                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
 205                         WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
 206                         // identical for all parallel stages
 207                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 208                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 209                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 210                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 211                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 212                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 213                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 214                         SKIP_ERRORPRONE = true
 215                     }
 216                     steps {
 217                         dir("${SOURCEDIR}") {
 218                             checkout scm
 219                         }
 220                         dir("${YETUSDIR}") {
 221                             checkout([
 222                               $class           : 'GitSCM',
 223                               branches         : [[name: "${YETUS_VERSION}"]],
 224                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 225                             )
 226                         }
 227                         dir("${WORKDIR}") {
 228                             withCredentials([
 229                               usernamePassword(
 230                                 credentialsId: 'apache-hbase-at-github.com',
 231                                 passwordVariable: 'GITHUB_PASSWORD',
 232                                 usernameVariable: 'GITHUB_USER'
 233                               )]) {
 234                                 script {
 235                                   def ret = sh(
 236                                     label: 'test-patch',
 237                                     returnStatus: true,
 238                                     script: '''#!/bin/bash -e
 239                                       hostname -a ; pwd ; ls -la
 240                                       printenv 2>&1 | sort
 241                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 242                                       "${YETUS_DRIVER}"
 243                                     '''
 244                                   )
 245                                   if (ret != 0) {
 246                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 247                                     // test output. See HBASE-26339 for more details.
 248                                     currentBuild.result = 'UNSTABLE'
 249                                   }
 250                                 }
 251                             }
 252                         }
 253                     }
 254                     post {
 255                         always {
 256                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
 257                               allowEmptyResults: true, skipPublishingChecks: true
 258                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 259                                 if [ -d "${PATCHDIR}/archiver" ]; then
 260                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 261                                   if [[ 0 -ne ${count} ]]; then
 262                                     echo "zipping ${count} archived files"
 263                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 264                                   else
 265                                     echo "No archived files, skipping compressing."
 266                                   fi
 267                                 else
 268                                   echo "No archiver directory, skipping compressing."
 269                                 fi
 270                             '''
 271                             sshPublisher(publishers: [
 272                               sshPublisherDesc(configName: 'Nightlies',
 273                                 transfers: [
 274                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 275                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 276                                   )
 277                                 ]
 278                               )
 279                             ])
 280                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 281                             sh '''#!/bin/bash -e
 282                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 283                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 284                               rm -rf "${PATCHDIR}/test_logs.zip"
 285                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
 286                             else
 287                               echo "No test_logs.zip, skipping"
 288                             fi
 289                             '''
 290                             // Has to be relative to WORKSPACE.
 291                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 292                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 293                             publishHTML target: [
 294                               allowMissing: true,
 295                               keepAll: true,
 296                               alwaysLinkToLastBuild: true,
 297                               // Has to be relative to WORKSPACE
 298                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 299                               reportFiles: 'report.html',
 300                               reportName: 'PR JDK8 Hadoop3 Check Report'
 301                             ]
 302                         }
 303                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 304                         cleanup() {
 305                             script {
 306                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 307                                     # See YETUS-764
 308                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 309                                       echo "test-patch process appears to still be running: killing"
 310                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 311                                       sleep 10
 312                                     fi
 313                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 314                                       echo "test-patch container appears to still be running: killing"
 315                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 316                                     fi
 317                                     # See HADOOP-13951
 318                                     chmod -R u+rxw "${WORKSPACE}"
 319                                 '''
 320                                 dir ("${WORKDIR}") {
 321                                     deleteDir()
 322                                 }
 323                             }
 324                         }
 325                     }
 326                 }
 327                 stage ('yetus jdk11 hadoop3 checks') {
 328                     agent {
 329                         node {
 330                             label 'hbase'
 331                         }
 332                     }
 333                     environment {
 334                         // customized per parallel stage
 335                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 336                         SET_JAVA_HOME = '/usr/lib/jvm/java-11'
 337                         WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
 338                         // identical for all parallel stages
 339                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 340                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 341                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 342                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 343                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 344                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 345                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 346                         SKIP_ERRORPRONE = true
 347                     }
 348                     steps {
 349                         dir("${SOURCEDIR}") {
 350                             checkout scm
 351                         }
 352                         dir("${YETUSDIR}") {
 353                             checkout([
 354                               $class           : 'GitSCM',
 355                               branches         : [[name: "${YETUS_VERSION}"]],
 356                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 357                             )
 358                         }
 359                         dir("${WORKDIR}") {
 360                             withCredentials([
 361                               usernamePassword(
 362                                 credentialsId: 'apache-hbase-at-github.com',
 363                                 passwordVariable: 'GITHUB_PASSWORD',
 364                                 usernameVariable: 'GITHUB_USER'
 365                               )]) {
 366                                 script {
 367                                   def ret = sh(
 368                                     label: 'test-patch',
 369                                     returnStatus: true,
 370                                     script: '''#!/bin/bash -e
 371                                       hostname -a ; pwd ; ls -la
 372                                       printenv 2>&1 | sort
 373                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 374                                       "${YETUS_DRIVER}"
 375                                     '''
 376                                   )
 377                                   if (ret != 0) {
 378                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 379                                     // test output. See HBASE-26339 for more details.
 380                                     currentBuild.result = 'UNSTABLE'
 381                                   }
 382                                 }
 383                             }
 384                         }
 385                     }
 386                     post {
 387                         always {
 388                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
 389                               allowEmptyResults: true, skipPublishingChecks: true
 390                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 391                                 if [ -d "${PATCHDIR}/archiver" ]; then
 392                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 393                                   if [[ 0 -ne ${count} ]]; then
 394                                     echo "zipping ${count} archived files"
 395                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 396                                   else
 397                                     echo "No archived files, skipping compressing."
 398                                   fi
 399                                 else
 400                                   echo "No archiver directory, skipping compressing."
 401                                 fi
 402                             '''
 403                             sshPublisher(publishers: [
 404                               sshPublisherDesc(configName: 'Nightlies',
 405                                 transfers: [
 406                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 407                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 408                                   )
 409                                 ]
 410                               )
 411                             ])
 412                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 413                             sh '''#!/bin/bash -e
 414                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 415                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 416                               rm -rf "${PATCHDIR}/test_logs.zip"
 417                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
 418                             else
 419                               echo "No test_logs.zip, skipping"
 420                             fi
 421                             '''
 422                             // Has to be relative to WORKSPACE.
 423                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 424                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 425                             publishHTML target: [
 426                               allowMissing: true,
 427                               keepAll: true,
 428                               alwaysLinkToLastBuild: true,
 429                               // Has to be relative to WORKSPACE
 430                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 431                               reportFiles: 'report.html',
 432                               reportName: 'PR JDK11 Hadoop3 Check Report'
 433                             ]
 434                         }
 435                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 436                         cleanup() {
 437                             script {
 438                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 439                                     # See YETUS-764
 440                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 441                                       echo "test-patch process appears to still be running: killing"
 442                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 443                                       sleep 10
 444                                     fi
 445                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 446                                       echo "test-patch container appears to still be running: killing"
 447                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 448                                     fi
 449                                     # See HADOOP-13951
 450                                     chmod -R u+rxw "${WORKSPACE}"
 451                                 '''
 452                                 dir ("${WORKDIR}") {
 453                                     deleteDir()
 454                                 }
 455                             }
 456                         }
 457                     }
 458                 }
 459             }
 460         }
 461     }
 462
 463     post {
 464         // Jenkins pipeline jobs fill slaves on PRs without this :(
 465         cleanup() {
 466             script {
 467                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 468                     # See HADOOP-13951
 469                     chmod -R u+rxw "${WORKSPACE}"
 470                     '''
 471                 deleteDir()
 472             }
 473         }
 474     }
 475 }