dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-19902 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=12500
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118     esac
 119   done
 120 }
 121
 122 ## @description  Queue up modules for this personality
 123 ## @audience     private
 124 ## @stability    evolving
 125 ## @param        repostatus
 126 ## @param        testtype
 127 function personality_modules
 128 {
 129   local repostatus=$1
 130   local testtype=$2
 131   local extra=""
 132   local branch1jdk8=()
 133   local jdk8module=""
 134   local MODULES=("${CHANGED_MODULES[@]}")
 135
 136   yetus_info "Personality: ${repostatus} ${testtype}"
 137
 138   clear_personality_queue
 139
 140   # At a few points, hbase modules can run build, test, etc. in parallel
 141   # Let it happen. Means we'll use more CPU but should be for short bursts.
 142   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 143   extra="--threads=2 -DHBasePatchProcess"
 144   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 145     extra="${extra} -Dhttps.protocols=TLSv1.2"
 146   fi
 147
 148   if [[ -n "${HADOOP_PROFILE}" ]]; then
 149     extra="${extra} -Phadoop-${HADOOP_PROFILE}"
 150   fi
 151
 152   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 153   # full source code instead. In this case, do full compiles, tests, etc instead of per
 154   # module.
 155   # Used in nightly runs.
 156   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 157   # modules if root is included. HBASE-18505
 158   if [[ "${BUILDMODE}" == "full" ]] || \
 159      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 160      [[ "${MODULES[*]}" =~ \. ]]; }; then
 161     MODULES=(.)
 162   fi
 163
 164   # If the checkstyle configs change, check everything.
 165   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 166     MODULES=(.)
 167   fi
 168
 169   if [[ ${testtype} == mvninstall ]]; then
 170     # shellcheck disable=SC2086
 171     personality_enqueue_module . ${extra}
 172     return
 173   fi
 174
 175   # This list should include any modules that require jdk8. Maven should be configured to only
 176   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 177   # module to build as yetus does if something changes in the module.  Rather than try to
 178   # figure out what jdk is in use so we can duplicate the module activation logic, just
 179   # build at the top level if anything changes in one of these modules and let maven sort it out.
 180   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 181   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 182     for jdk8module in "${branch1jdk8[@]}"; do
 183       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 184         MODULES=(.)
 185         break
 186       fi
 187     done
 188   fi
 189
 190   if [[ ${testtype} == spotbugs ]]; then
 191     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 192     # report new warnings for changed modules only.
 193     # For some reason, spotbugs on root is not working, but running on individual modules is
 194     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 195     for module in "${CHANGED_MODULES[@]}"; do
 196       # skip spotbugs on any module that lacks content in `src/main/java`
 197       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 198           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 199         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 200         continue
 201       else
 202         # shellcheck disable=SC2086
 203         personality_enqueue_module ${module} ${extra}
 204       fi
 205     done
 206     return
 207   fi
 208
 209   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 210       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 211     extra="${extra} -PerrorProne"
 212   fi
 213
 214   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 215   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 216   # tests respectively.
 217   if [[ ${testtype} == unit ]]; then
 218     local tests_arg=""
 219     get_include_exclude_tests_arg tests_arg
 220     extra="${extra} -PrunAllTests ${tests_arg}"
 221
 222     # Inject the jenkins build-id for our surefire invocations
 223     # Used by zombie detection stuff, even though we're not including that yet.
 224     if [ -n "${BUILD_ID}" ]; then
 225       extra="${extra} -Dbuild.id=${BUILD_ID}"
 226     fi
 227
 228     # If the set of changed files includes CommonFSUtils then add the hbase-server
 229     # module to the set of modules (if not already included) to be tested
 230     for f in "${CHANGED_FILES[@]}"
 231     do
 232       if [[ "${f}" =~ CommonFSUtils ]]; then
 233         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 234           MODULES+=("hbase-server")
 235         fi
 236         break
 237       fi
 238     done
 239   fi
 240
 241   for module in "${MODULES[@]}"; do
 242     # shellcheck disable=SC2086
 243     personality_enqueue_module ${module} ${extra}
 244   done
 245 }
 246
 247 ## @description places where we override the built in assumptions about what tests to run
 248 ## @audience    private
 249 ## @stability   evolving
 250 ## @param       filename of changed file
 251 function personality_file_tests
 252 {
 253   local filename=$1
 254   yetus_debug "HBase specific personality_file_tests"
 255   # If the change is to the refguide, then we don't need any builtin yetus tests
 256   # the refguide test (below) will suffice for coverage.
 257   if [[ ${filename} =~ src/main/asciidoc ]] ||
 258      [[ ${filename} =~ src/main/xslt ]]; then
 259     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 260   else
 261     # If we change our asciidoc, rebuild mvnsite
 262     if [[ ${BUILDTOOL} = maven ]]; then
 263       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 264         yetus_debug "tests/mvnsite: ${filename}"
 265         add_test mvnsite
 266       fi
 267     fi
 268     # If we change checkstyle configs, run checkstyle
 269     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 270       yetus_debug "tests/checkstyle: ${filename}"
 271       add_test checkstyle
 272     fi
 273     # fallback to checking which tests based on what yetus would do by default
 274     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 275       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 276     elif declare -f builtin_personality_file_tests >/dev/null; then
 277       builtin_personality_file_tests "${filename}"
 278     fi
 279   fi
 280 }
 281
 282 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 283 #                tests and sets given variable to arguments to be passes to maven command.
 284 ## @audience     private
 285 ## @stability    evolving
 286 ## @param        name of variable to set with maven arguments
 287 function get_include_exclude_tests_arg
 288 {
 289   local  __resultvar=$1
 290   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 291   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 292   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 293       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 294         excludes=$(cat excludes)
 295         yetus_debug "excludes=${excludes}"
 296         if [[ -n "${excludes}" ]]; then
 297           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 298         fi
 299         rm excludes
 300       else
 301         yetus_error "Wget error $? in fetching excludes file from url" \
 302              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 303       fi
 304   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 305       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 306         includes=$(cat includes)
 307         yetus_debug "includes=${includes}"
 308         if [[ -n "${includes}" ]]; then
 309           eval "${__resultvar}='-Dtest=${includes}'"
 310         fi
 311         rm includes
 312       else
 313         yetus_error "Wget error $? in fetching includes file from url" \
 314              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 315       fi
 316   else
 317     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 318     FLAKY_URL="https://builds.apache.org/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
 319     if wget "${FLAKY_URL}" -O "excludes"; then
 320       excludes=$(cat excludes)
 321         yetus_debug "excludes=${excludes}"
 322         if [[ -n "${excludes}" ]]; then
 323           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 324         fi
 325         rm excludes
 326       else
 327         yetus_error "Wget error $? in fetching excludes file from url" \
 328              "${FLAKY_URL}. Ignoring and proceeding."
 329       fi
 330   fi
 331 }
 332
 333 ###################################################
 334 # Below here are our one-off tests specific to hbase.
 335 # TODO break them into individual files so it's easier to maintain them?
 336
 337 # TODO line length check? could ignore all java files since checkstyle gets them.
 338
 339 ###################################################
 340
 341 add_test_type refguide
 342
 343 function refguide_initialize
 344 {
 345   maven_add_install refguide
 346 }
 347
 348 function refguide_filefilter
 349 {
 350   local filename=$1
 351
 352   if [[ ${filename} =~ src/main/asciidoc ]] ||
 353      [[ ${filename} =~ src/main/xslt ]] ||
 354      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default.xml ]]; then
 355     add_test refguide
 356   fi
 357 }
 358
 359 function refguide_rebuild
 360 {
 361   local repostatus=$1
 362   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 363   declare -i count
 364   declare pdf_output
 365
 366   if ! verify_needed_test refguide; then
 367     return 0
 368   fi
 369
 370   big_console_header "Checking we can create the ref guide on ${repostatus}"
 371
 372   start_clock
 373
 374   # disabled because "maven_executor" needs to return both command and args
 375   # shellcheck disable=2046
 376   echo_and_redirect "${logfile}" \
 377     $(maven_executor) clean site --batch-mode \
 378       -pl . \
 379       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 380       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 381
 382   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 383   if [[ ${count} -gt 0 ]]; then
 384     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 385     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 386     return 1
 387   fi
 388
 389   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 390     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 391     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 392     return 1
 393   fi
 394
 395   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 396     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 397     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 398     return 1
 399   fi
 400
 401   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 402     pdf_output="book.pdf"
 403   else
 404     pdf_output="apache_hbase_reference_guide.pdf"
 405   fi
 406
 407   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 408     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 409     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 410     return 1
 411   fi
 412
 413   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 414   add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 415   return 0
 416 }
 417
 418 add_test_type shadedjars
 419
 420
 421 function shadedjars_initialize
 422 {
 423   yetus_debug "initializing shaded client checks."
 424   maven_add_install shadedjars
 425 }
 426
 427 ## @description  only run the test if java changes.
 428 ## @audience     private
 429 ## @stability    evolving
 430 ## @param        filename
 431 function shadedjars_filefilter
 432 {
 433   local filename=$1
 434
 435   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 436     add_test shadedjars
 437   fi
 438 }
 439
 440 ## @description test the shaded client artifacts
 441 ## @audience private
 442 ## @stability evolving
 443 ## @param repostatus
 444 function shadedjars_rebuild
 445 {
 446   local repostatus=$1
 447   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 448
 449   if ! verify_needed_test shadedjars; then
 450     return 0
 451   fi
 452
 453   big_console_header "Checking shaded client builds on ${repostatus}"
 454
 455   start_clock
 456
 457   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 458     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 459     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 460     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 461   if [[ -n "${HADOOP_PROFILE}" ]]; then
 462     maven_args+=("-Phadoop-${HADOOP_PROFILE}")
 463   fi
 464
 465   # disabled because "maven_executor" needs to return both command and args
 466   # shellcheck disable=2046
 467   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 468
 469   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 470   if [[ ${count} -gt 0 ]]; then
 471     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 472     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 473     return 1
 474   fi
 475
 476   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 477   return 0
 478 }
 479
 480 ###################################################
 481
 482 add_test_type hadoopcheck
 483
 484 ## @description  hadoopcheck file filter
 485 ## @audience     private
 486 ## @stability    evolving
 487 ## @param        filename
 488 function hadoopcheck_filefilter
 489 {
 490   local filename=$1
 491
 492   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 493     add_test hadoopcheck
 494   fi
 495 }
 496
 497 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 498 ## @audience     private
 499 ## @stability    evolving
 500 function hadoopcheck_parse_args
 501 {
 502   declare i
 503
 504   for i in "$@"; do
 505     case ${i} in
 506       --quick-hadoopcheck)
 507         delete_parameter "${i}"
 508         QUICK_HADOOPCHECK=true
 509       ;;
 510     esac
 511   done
 512 }
 513
 514 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 515 ## @audience     private
 516 ## @stability    evolving
 517 function hadoopcheck_docker_support
 518 {
 519   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 520 }
 521
 522 ## @description  hadoopcheck test
 523 ## @audience     private
 524 ## @stability    evolving
 525 ## @param        repostatus
 526 function hadoopcheck_rebuild
 527 {
 528   local repostatus=$1
 529   local hadoopver
 530   local logfile
 531   local count
 532   local result=0
 533   local hbase_hadoop2_versions
 534   local hbase_hadoop3_versions
 535
 536   if [[ "${repostatus}" = branch ]]; then
 537     return 0
 538   fi
 539
 540   if ! verify_needed_test hadoopcheck; then
 541     return 0
 542   fi
 543
 544   big_console_header "Compiling against various Hadoop versions"
 545
 546   start_clock
 547
 548   # All supported Hadoop versions that we want to test the compilation with
 549   # See the Hadoop section on prereqs in the HBase Reference Guide
 550   if [[ "${PATCH_BRANCH}" = branch-1.3 ]]; then
 551     yetus_info "Setting Hadoop 2 versions to test based on branch-1.3 rules."
 552     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 553       hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
 554     else
 555       hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 556     fi
 557   elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 558     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 559     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 560       hbase_hadoop2_versions="2.7.7"
 561     else
 562       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 563     fi
 564   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 565     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 566     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 567       hbase_hadoop2_versions="2.8.5 2.9.2"
 568     else
 569       hbase_hadoop2_versions="2.8.5 2.9.2"
 570     fi
 571   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 572     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 573     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 574       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 575     else
 576       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 577     fi
 578   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 579     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 580     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 581       hbase_hadoop2_versions="2.7.7 2.8.5"
 582     else
 583       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 584     fi
 585   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 586     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 587     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 588       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 589     else
 590       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 591     fi
 592   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 593     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 594     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 595       hbase_hadoop2_versions="2.10.0"
 596     else
 597       hbase_hadoop2_versions="2.10.0"
 598     fi
 599   else
 600     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 601     hbase_hadoop2_versions=""
 602   fi
 603   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 604     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 605     hbase_hadoop3_versions=""
 606   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 607     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 608     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 609       hbase_hadoop3_versions="3.0.3 3.1.2"
 610     else
 611       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 612     fi
 613   else
 614     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
 615     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 616       hbase_hadoop3_versions="3.1.2 3.2.1"
 617     else
 618       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1"
 619     fi
 620   fi
 621
 622   export MAVEN_OPTS="${MAVEN_OPTS}"
 623   for hadoopver in ${hbase_hadoop2_versions}; do
 624     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 625     # disabled because "maven_executor" needs to return both command and args
 626     # shellcheck disable=2046
 627     echo_and_redirect "${logfile}" \
 628       $(maven_executor) clean install \
 629         -DskipTests -DHBasePatchProcess \
 630         -Dhadoop-two.version="${hadoopver}"
 631     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 632     if [[ ${count} -gt 0 ]]; then
 633       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 634       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 635       ((result=result+1))
 636     fi
 637   done
 638
 639   for hadoopver in ${hbase_hadoop3_versions}; do
 640     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 641     # disabled because "maven_executor" needs to return both command and args
 642     # shellcheck disable=2046
 643     echo_and_redirect "${logfile}" \
 644       $(maven_executor) clean install \
 645         -DskipTests -DHBasePatchProcess \
 646         -Dhadoop-three.version="${hadoopver}" \
 647         -Phadoop-3.0
 648     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 649     if [[ ${count} -gt 0 ]]; then
 650       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 651       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 652       ((result=result+1))
 653     fi
 654   done
 655
 656   if [[ ${result} -gt 0 ]]; then
 657     return 1
 658   fi
 659
 660   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 661     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 662       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 663     else
 664       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 665     fi
 666   else
 667     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 668   fi
 669
 670   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 671   echo_and_redirect "${logfile}" \
 672     $(maven_executor) clean install \
 673       -DskipTests -DHBasePatchProcess
 674
 675   return 0
 676 }
 677
 678 ######################################
 679
 680 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 681 add_test_type hbaseprotoc
 682
 683 ## @description  hbaseprotoc file filter
 684 ## @audience     private
 685 ## @stability    evolving
 686 ## @param        filename
 687 function hbaseprotoc_filefilter
 688 {
 689   local filename=$1
 690
 691   if [[ ${filename} =~ \.proto$ ]]; then
 692     add_test hbaseprotoc
 693   fi
 694 }
 695
 696 ## @description  check hbase proto compilation
 697 ## @audience     private
 698 ## @stability    evolving
 699 ## @param        repostatus
 700 function hbaseprotoc_rebuild
 701 {
 702   declare repostatus=$1
 703   declare i=0
 704   declare fn
 705   declare module
 706   declare logfile
 707   declare count
 708   declare result
 709
 710   if [[ "${repostatus}" = branch ]]; then
 711     return 0
 712   fi
 713
 714   if ! verify_needed_test hbaseprotoc; then
 715     return 0
 716   fi
 717
 718   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 719
 720   start_clock
 721
 722   personality_modules patch hbaseprotoc
 723   # Need to run 'install' instead of 'compile' because shading plugin
 724   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 725   # half of its process done.
 726   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 727
 728   # shellcheck disable=SC2153
 729   until [[ $i -eq "${#MODULE[@]}" ]]; do
 730     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 731       ((result=result+1))
 732       ((i=i+1))
 733       continue
 734     fi
 735     module=${MODULE[$i]}
 736     fn=$(module_file_fragment "${module}")
 737     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 738
 739     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 740
 741     if [[ ${count} -gt 0 ]]; then
 742       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 743         "${count} new protoc errors in ${module}."
 744       ((result=result+1))
 745     fi
 746     ((i=i+1))
 747   done
 748
 749   modules_messages patch hbaseprotoc true
 750   if [[ ${result} -gt 0 ]]; then
 751     return 1
 752   fi
 753   return 0
 754 }
 755
 756 ######################################
 757
 758 add_test_type hbaseanti
 759
 760 ## @description  hbaseanti file filter
 761 ## @audience     private
 762 ## @stability    evolving
 763 ## @param        filename
 764 function hbaseanti_filefilter
 765 {
 766   local filename=$1
 767
 768   if [[ ${filename} =~ \.java$ ]]; then
 769     add_test hbaseanti
 770   fi
 771 }
 772
 773 ## @description  hbaseanti patch file check
 774 ## @audience     private
 775 ## @stability    evolving
 776 ## @param        filename
 777 function hbaseanti_patchfile
 778 {
 779   local patchfile=$1
 780   local warnings
 781   local result
 782
 783   if [[ "${BUILDMODE}" = full ]]; then
 784     return 0
 785   fi
 786
 787   if ! verify_needed_test hbaseanti; then
 788     return 0
 789   fi
 790
 791   big_console_header "Checking for known anti-patterns"
 792
 793   start_clock
 794
 795   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 796   if [[ ${warnings} -gt 0 ]]; then
 797     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 798     ((result=result+1))
 799   fi
 800
 801   if [[ ${result} -gt 0 ]]; then
 802     return 1
 803   fi
 804
 805   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 806   return 0
 807 }
 808
 809 ## @description  process the javac output for generating WARNING/ERROR
 810 ## @audience     private
 811 ## @stability    evolving
 812 ## @param        input filename
 813 ## @param        output filename
 814 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 815 # This is because that the output order of the error prone warnings is not stable, so the diff
 816 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 817 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 818 # really hard to sort by file name first and then line number and column number in shell...
 819 function hbase_javac_logfilter
 820 {
 821   declare input=$1
 822   declare output=$2
 823
 824   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 825 }
 826
 827 ## This is named so that yetus will check us right after running tests.
 828 ## Essentially, we check for normal failures and then we look for zombies.
 829 #function hbase_unit_logfilter
 830 #{
 831 #  declare testtype="unit"
 832 #  declare input=$1
 833 #  declare output=$2
 834 #  declare processes
 835 #  declare process_output
 836 #  declare zombies
 837 #  declare zombie_count=0
 838 #  declare zombie_process
 839 #
 840 #  yetus_debug "in hbase-specific unit logfilter."
 841 #
 842 #  # pass-through to whatever is counting actual failures
 843 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 844 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 845 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 846 #    "${testtype}_logfilter" "${input}" "${output}"
 847 #  fi
 848 #
 849 #  start_clock
 850 #  if [ -n "${BUILD_ID}" ]; then
 851 #    yetus_debug "Checking for zombie test processes."
 852 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 853 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 854 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 855 #      yetus_debug "${processes}"
 856 #      sleep 30
 857 #      #shellcheck disable=SC2016
 858 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 859 #        # Test our zombie still running (and that it still an hbase build item)
 860 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 861 #        if [[ -n "${process_output}" ]]; then
 862 #          yetus_error "Zombie: ${process_output}"
 863 #          ((zombie_count = zombie_count + 1))
 864 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 865 #          zombies="${zombies} ${zombie_process}"
 866 #        fi
 867 #      done
 868 #    fi
 869 #    if [ "${zombie_count}" -ne 0 ]; then
 870 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 871 #      populate_test_table "zombie unit tests" "${zombies}"
 872 #    else
 873 #      yetus_info "Zombie check complete. All test runs exited normally."
 874 #      stop_clock
 875 #    fi
 876 #  else
 877 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 878 #  fi
 879 #
 880 #}