dev-support/Jenkinsfile_GitHub

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 //   http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing,
  12 // software distributed under the License is distributed on an
  13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 // KIND, either express or implied.  See the License for the
  15 // specific language governing permissions and limitations
  16 // under the License.
  17
  18 pipeline {
  19
  20     agent {
  21         label 'hbase'
  22     }
  23
  24     options {
  25         // N.B. this is per-branch, which means per PR
  26         disableConcurrentBuilds()
  27         buildDiscarder(logRotator(numToKeepStr: '15'))
  28         timeout (time: 10, unit: 'HOURS')
  29         timestamps()
  30         skipDefaultCheckout()
  31     }
  32
  33     environment {
  34         SRC_REL = 'src'
  35         PATCH_REL = 'output'
  36         YETUS_REL = 'yetus'
  37         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  38         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
  39         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
  40         YETUS_VERSION = 'rel/0.12.0'
  41         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  42         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
  43         // output from surefire; sadly the archive function in yetus only works on file names.
  44         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  45         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
  46         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
  47         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  48
  49         // a global view of paths. parallel stages can land on the same host concurrently, so each
  50         // stage works in its own subdirectory. there is an "output" under each of these
  51         // directories, which we retrieve after the build is complete.
  52         WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
  53         WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
  54         WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  55         ASF_NIGHTLIES = 'https://nightlies.apache.org'
  56         ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
  57         ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
  58     }
  59
  60     parameters {
  61         booleanParam(name: 'DEBUG',
  62                defaultValue: false,
  63                description: 'Print extra outputs for debugging the jenkins job and yetus')
  64     }
  65
  66     stages {
  67         stage ('precommit checks') {
  68             parallel {
  69                 stage ('yetus general check') {
  70                     agent {
  71                         node {
  72                             label 'hbase'
  73                         }
  74                     }
  75                     environment {
  76                         // customized per parallel stage
  77                         PLUGINS = "${GENERAL_CHECK_PLUGINS}"
  78                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
  79                         WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  80                         // identical for all parallel stages
  81                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
  82                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
  83                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
  84                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
  85                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
  86                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
  87                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
  88                         ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}"
  89                     }
  90                     steps {
  91                         dir("${SOURCEDIR}") {
  92                             checkout scm
  93                         }
  94                         dir("${YETUSDIR}") {
  95                             checkout([
  96                               $class           : 'GitSCM',
  97                               branches         : [[name: "${YETUS_VERSION}"]],
  98                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
  99                             )
 100                         }
 101                         dir("${WORKDIR}") {
 102                             withCredentials([
 103                                 usernamePassword(
 104                                   credentialsId: 'apache-hbase-at-github.com',
 105                                   passwordVariable: 'GITHUB_PASSWORD',
 106                                   usernameVariable: 'GITHUB_USER'
 107                                 )]) {
 108                                 script {
 109                                   def ret = sh(
 110                                     label: 'test-patch',
 111                                     returnStatus: true,
 112                                     script: '''#!/bin/bash -e
 113                                       hostname -a ; pwd ; ls -la
 114                                       printenv 2>&1 | sort
 115                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 116                                       "${YETUS_DRIVER}"
 117                                     '''
 118                                   )
 119                                   if (ret != 0) {
 120                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 121                                     // test output. See HBASE-26339 for more details.
 122                                     currentBuild.result = 'UNSTABLE'
 123                                   }
 124                                 }
 125                             }
 126                         }
 127                     }
 128                     post {
 129                         always {
 130                             sshPublisher(publishers: [
 131                               sshPublisherDesc(configName: 'Nightlies',
 132                                 transfers: [
 133                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 134                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*"
 135                                   )
 136                                 ]
 137                               )
 138                             ])
 139                             sh '''#!/bin/bash -e
 140                             if [ -d "${PATCHDIR}/branch-site" ]; then
 141                               echo "Remove ${PATCHDIR}/branch-site for saving space"
 142                               rm -rf "${PATCHDIR}/branch-site"
 143                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
 144                             else
 145                               echo "No branch-site, skipping"
 146                             fi
 147                             if [ -d "${PATCHDIR}/patch-site" ]; then
 148                               echo "Remove ${PATCHDIR}/patch-site for saving space"
 149                               rm -rf "${PATCHDIR}/patch-site"
 150                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
 151                             else
 152                               echo "No patch-site, skipping"
 153                             fi
 154                             '''
 155                             // Has to be relative to WORKSPACE.
 156                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*"
 157                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*"
 158                             publishHTML target: [
 159                               allowMissing: true,
 160                               keepAll: true,
 161                               alwaysLinkToLastBuild: true,
 162                               // Has to be relative to WORKSPACE
 163                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 164                               reportFiles: 'report.html',
 165                               reportName: 'PR General Check Report'
 166                             ]
 167                         }
 168                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 169                         cleanup() {
 170                             script {
 171                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 172                                     # See YETUS-764
 173                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 174                                       echo "test-patch process appears to still be running: killing"
 175                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 176                                       sleep 10
 177                                     fi
 178                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 179                                       echo "test-patch container appears to still be running: killing"
 180                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 181                                     fi
 182                                     # See HADOOP-13951
 183                                     chmod -R u+rxw "${WORKSPACE}"
 184                                 '''
 185                                 dir ("${WORKDIR}") {
 186                                     deleteDir()
 187                                 }
 188                             }
 189                         }
 190                     }
 191                 }
 192                 stage ('yetus jdk8 Hadoop3 checks') {
 193                     agent {
 194                         node {
 195                             label 'hbase'
 196                         }
 197                     }
 198                     environment {
 199                         // customized per parallel stage
 200                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 201                         SET_JAVA_HOME = '/usr/lib/jvm/java-8'
 202                         WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
 203                         // identical for all parallel stages
 204                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 205                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 206                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 207                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 208                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 209                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 210                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 211                         SKIP_ERRORPRONE = true
 212                     }
 213                     steps {
 214                         dir("${SOURCEDIR}") {
 215                             checkout scm
 216                         }
 217                         dir("${YETUSDIR}") {
 218                             checkout([
 219                               $class           : 'GitSCM',
 220                               branches         : [[name: "${YETUS_VERSION}"]],
 221                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 222                             )
 223                         }
 224                         dir("${WORKDIR}") {
 225                             withCredentials([
 226                               usernamePassword(
 227                                 credentialsId: 'apache-hbase-at-github.com',
 228                                 passwordVariable: 'GITHUB_PASSWORD',
 229                                 usernameVariable: 'GITHUB_USER'
 230                               )]) {
 231                                 script {
 232                                   def ret = sh(
 233                                     label: 'test-patch',
 234                                     returnStatus: true,
 235                                     script: '''#!/bin/bash -e
 236                                       hostname -a ; pwd ; ls -la
 237                                       printenv 2>&1 | sort
 238                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 239                                       "${YETUS_DRIVER}"
 240                                     '''
 241                                   )
 242                                   if (ret != 0) {
 243                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 244                                     // test output. See HBASE-26339 for more details.
 245                                     currentBuild.result = 'UNSTABLE'
 246                                   }
 247                                 }
 248                             }
 249                         }
 250                     }
 251                     post {
 252                         always {
 253                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
 254                               allowEmptyResults: true, skipPublishingChecks: true
 255                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 256                                 if [ -d "${PATCHDIR}/archiver" ]; then
 257                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 258                                   if [[ 0 -ne ${count} ]]; then
 259                                     echo "zipping ${count} archived files"
 260                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 261                                   else
 262                                     echo "No archived files, skipping compressing."
 263                                   fi
 264                                 else
 265                                   echo "No archiver directory, skipping compressing."
 266                                 fi
 267                             '''
 268                             sshPublisher(publishers: [
 269                               sshPublisherDesc(configName: 'Nightlies',
 270                                 transfers: [
 271                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 272                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 273                                   )
 274                                 ]
 275                               )
 276                             ])
 277                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 278                             sh '''#!/bin/bash -e
 279                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 280                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 281                               rm -rf "${PATCHDIR}/test_logs.zip"
 282                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
 283                             else
 284                               echo "No test_logs.zip, skipping"
 285                             fi
 286                             '''
 287                             // Has to be relative to WORKSPACE.
 288                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 289                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 290                             publishHTML target: [
 291                               allowMissing: true,
 292                               keepAll: true,
 293                               alwaysLinkToLastBuild: true,
 294                               // Has to be relative to WORKSPACE
 295                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 296                               reportFiles: 'report.html',
 297                               reportName: 'PR JDK8 Hadoop3 Check Report'
 298                             ]
 299                         }
 300                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 301                         cleanup() {
 302                             script {
 303                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 304                                     # See YETUS-764
 305                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 306                                       echo "test-patch process appears to still be running: killing"
 307                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 308                                       sleep 10
 309                                     fi
 310                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 311                                       echo "test-patch container appears to still be running: killing"
 312                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 313                                     fi
 314                                     # See HADOOP-13951
 315                                     chmod -R u+rxw "${WORKSPACE}"
 316                                 '''
 317                                 dir ("${WORKDIR}") {
 318                                     deleteDir()
 319                                 }
 320                             }
 321                         }
 322                     }
 323                 }
 324                 stage ('yetus jdk11 hadoop3 checks') {
 325                     agent {
 326                         node {
 327                             label 'hbase'
 328                         }
 329                     }
 330                     environment {
 331                         // customized per parallel stage
 332                         PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
 333                         SET_JAVA_HOME = '/usr/lib/jvm/java-11'
 334                         WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
 335                         // identical for all parallel stages
 336                         WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
 337                         YETUSDIR = "${WORKDIR}/${YETUS_REL}"
 338                         SOURCEDIR = "${WORKDIR}/${SRC_REL}"
 339                         PATCHDIR = "${WORKDIR}/${PATCH_REL}"
 340                         BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
 341                         DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
 342                         YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
 343                         SKIP_ERRORPRONE = true
 344                     }
 345                     steps {
 346                         dir("${SOURCEDIR}") {
 347                             checkout scm
 348                         }
 349                         dir("${YETUSDIR}") {
 350                             checkout([
 351                               $class           : 'GitSCM',
 352                               branches         : [[name: "${YETUS_VERSION}"]],
 353                               userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
 354                             )
 355                         }
 356                         dir("${WORKDIR}") {
 357                             withCredentials([
 358                               usernamePassword(
 359                                 credentialsId: 'apache-hbase-at-github.com',
 360                                 passwordVariable: 'GITHUB_PASSWORD',
 361                                 usernameVariable: 'GITHUB_USER'
 362                               )]) {
 363                                 script {
 364                                   def ret = sh(
 365                                     label: 'test-patch',
 366                                     returnStatus: true,
 367                                     script: '''#!/bin/bash -e
 368                                       hostname -a ; pwd ; ls -la
 369                                       printenv 2>&1 | sort
 370                                       echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
 371                                       "${YETUS_DRIVER}"
 372                                     '''
 373                                   )
 374                                   if (ret != 0) {
 375                                     // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
 376                                     // test output. See HBASE-26339 for more details.
 377                                     currentBuild.result = 'UNSTABLE'
 378                                   }
 379                                 }
 380                             }
 381                         }
 382                     }
 383                     post {
 384                         always {
 385                             junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
 386                               allowEmptyResults: true, skipPublishingChecks: true
 387                             sh label: 'zip surefire reports', script: '''#!/bin/bash -e
 388                                 if [ -d "${PATCHDIR}/archiver" ]; then
 389                                   count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
 390                                   if [[ 0 -ne ${count} ]]; then
 391                                     echo "zipping ${count} archived files"
 392                                     zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
 393                                   else
 394                                     echo "No archived files, skipping compressing."
 395                                   fi
 396                                 else
 397                                   echo "No archiver directory, skipping compressing."
 398                                 fi
 399                             '''
 400                             sshPublisher(publishers: [
 401                               sshPublisherDesc(configName: 'Nightlies',
 402                                 transfers: [
 403                                   sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
 404                                     sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
 405                                   )
 406                                 ]
 407                               )
 408                             ])
 409                             // remove the big test logs zip file, store the nightlies url in test_logs.txt
 410                             sh '''#!/bin/bash -e
 411                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
 412                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
 413                               rm -rf "${PATCHDIR}/test_logs.zip"
 414                               python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
 415                             else
 416                               echo "No test_logs.zip, skipping"
 417                             fi
 418                             '''
 419                             // Has to be relative to WORKSPACE.
 420                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
 421                             archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
 422                             publishHTML target: [
 423                               allowMissing: true,
 424                               keepAll: true,
 425                               alwaysLinkToLastBuild: true,
 426                               // Has to be relative to WORKSPACE
 427                               reportDir: "${WORKDIR_REL}/${PATCH_REL}",
 428                               reportFiles: 'report.html',
 429                               reportName: 'PR JDK11 Hadoop3 Check Report'
 430                             ]
 431                         }
 432                         // Jenkins pipeline jobs fill slaves on PRs without this :(
 433                         cleanup() {
 434                             script {
 435                                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 436                                     # See YETUS-764
 437                                     if [ -f "${PATCHDIR}/pidfile.txt" ]; then
 438                                       echo "test-patch process appears to still be running: killing"
 439                                       kill `cat "${PATCHDIR}/pidfile.txt"` || true
 440                                       sleep 10
 441                                     fi
 442                                     if [ -f "${PATCHDIR}/cidfile.txt" ]; then
 443                                       echo "test-patch container appears to still be running: killing"
 444                                       docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
 445                                     fi
 446                                     # See HADOOP-13951
 447                                     chmod -R u+rxw "${WORKSPACE}"
 448                                 '''
 449                                 dir ("${WORKDIR}") {
 450                                     deleteDir()
 451                                 }
 452                             }
 453                         }
 454                     }
 455                 }
 456             }
 457         }
 458     }
 459
 460     post {
 461         // Jenkins pipeline jobs fill slaves on PRs without this :(
 462         cleanup() {
 463             script {
 464                 sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
 465                     # See HADOOP-13951
 466                     chmod -R u+rxw "${WORKSPACE}"
 467                     '''
 468                 deleteDir()
 469             }
 470         }
 471     }
 472 }