dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Override the maven options
  83   MAVEN_OPTS="${MAVEN_OPTS:-"-Xms4G -Xmx4G"}"
  84
  85   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  86   # Up it. See HBASE-19902 for how we arrived at this number.
  87   #shellcheck disable=SC2034
  88   PROCLIMIT=10000
  89
  90   # Set docker container to run with 20g. Default is 4g in yetus.
  91   # See HBASE-19902 for how we arrived at 20g.
  92   #shellcheck disable=SC2034
  93   DOCKERMEMLIMIT=20g
  94 }
  95
  96 ## @description  Parse extra arguments required by personalities, if any.
  97 ## @audience     private
  98 ## @stability    evolving
  99 function personality_parse_args
 100 {
 101   declare i
 102
 103   for i in "$@"; do
 104     case ${i} in
 105       --exclude-tests-url=*)
 106         delete_parameter "${i}"
 107         EXCLUDE_TESTS_URL=${i#*=}
 108       ;;
 109       --include-tests-url=*)
 110         delete_parameter "${i}"
 111         INCLUDE_TESTS_URL=${i#*=}
 112       ;;
 113       --hadoop-profile=*)
 114         delete_parameter "${i}"
 115         HADOOP_PROFILE=${i#*=}
 116       ;;
 117       --skip-errorprone)
 118         delete_parameter "${i}"
 119         SKIP_ERRORPRONE=true
 120       ;;
 121     esac
 122   done
 123 }
 124
 125 ## @description  Queue up modules for this personality
 126 ## @audience     private
 127 ## @stability    evolving
 128 ## @param        repostatus
 129 ## @param        testtype
 130 function personality_modules
 131 {
 132   local repostatus=$1
 133   local testtype=$2
 134   local extra=""
 135   local branch1jdk8=()
 136   local jdk8module=""
 137   local MODULES=("${CHANGED_MODULES[@]}")
 138
 139   yetus_info "Personality: ${repostatus} ${testtype}"
 140
 141   clear_personality_queue
 142
 143   extra="-DHBasePatchProcess"
 144   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 145     extra="${extra} -Dhttps.protocols=TLSv1.2"
 146   fi
 147
 148   if [[ -n "${HADOOP_PROFILE}" ]]; then
 149     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 150   fi
 151
 152   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 153   # full source code instead. In this case, do full compiles, tests, etc instead of per
 154   # module.
 155   # Used in nightly runs.
 156   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 157   # modules if root is included. HBASE-18505
 158   if [[ "${BUILDMODE}" == "full" ]] || \
 159      ( ( [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]] ) && \
 160      [[ "${MODULES[*]}" =~ \. ]] ); then
 161     MODULES=(.)
 162   fi
 163
 164   # If the checkstyle configs change, check everything.
 165   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 166     MODULES=(.)
 167   fi
 168
 169   if [[ ${testtype} == mvninstall ]]; then
 170     # shellcheck disable=SC2086
 171     personality_enqueue_module . ${extra}
 172     return
 173   fi
 174
 175   # This list should include any modules that require jdk8. Maven should be configured to only
 176   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 177   # module to build as yetus does if something changes in the module.  Rather than try to
 178   # figure out what jdk is in use so we can duplicate the module activation logic, just
 179   # build at the top level if anything changes in one of these modules and let maven sort it out.
 180   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 181   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 182     for jdk8module in "${branch1jdk8[@]}"; do
 183       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 184         MODULES=(.)
 185         break
 186       fi
 187     done
 188   fi
 189
 190   if [[ ${testtype} == findbugs ]]; then
 191     # Run findbugs on each module individually to diff pre-patch and post-patch results and
 192     # report new warnings for changed modules only.
 193     # For some reason, findbugs on root is not working, but running on individual modules is
 194     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 195     for module in "${CHANGED_MODULES[@]}"; do
 196       # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
 197       # in src/main/java where findbugs goes to look
 198       if [[ ${module} == hbase-shell ]]; then
 199         continue
 200       elif [[ ${module} == hbase-it ]]; then
 201         continue
 202       else
 203         # shellcheck disable=SC2086
 204         personality_enqueue_module ${module} ${extra}
 205       fi
 206     done
 207     return
 208   fi
 209
 210   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 211       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 212     extra="${extra} -PerrorProne"
 213   fi
 214
 215   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 216   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 217   # tests respectively.
 218   if [[ ${testtype} == unit ]]; then
 219     local tests_arg=""
 220     get_include_exclude_tests_arg tests_arg
 221     extra="${extra} -PrunAllTests ${tests_arg}"
 222
 223     # Inject the jenkins build-id for our surefire invocations
 224     # Used by zombie detection stuff, even though we're not including that yet.
 225     if [ -n "${BUILD_ID}" ]; then
 226       extra="${extra} -Dbuild.id=${BUILD_ID}"
 227     fi
 228
 229     # If the set of changed files includes CommonFSUtils then add the hbase-server
 230     # module to the set of modules (if not already included) to be tested
 231     for f in "${CHANGED_FILES[@]}"
 232     do
 233       if [[ "${f}" =~ CommonFSUtils ]]; then
 234         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 235           MODULES+=("hbase-server")
 236         fi
 237         break
 238       fi
 239     done
 240   fi
 241
 242   for module in "${MODULES[@]}"; do
 243     # shellcheck disable=SC2086
 244     personality_enqueue_module ${module} ${extra}
 245   done
 246 }
 247
 248 ## @description places where we override the built in assumptions about what tests to run
 249 ## @audience    private
 250 ## @stability   evolving
 251 ## @param       filename of changed file
 252 function personality_file_tests
 253 {
 254   local filename=$1
 255   yetus_debug "HBase specific personality_file_tests"
 256   # If the change is to the refguide, then we don't need any builtin yetus tests
 257   # the refguide test (below) will suffice for coverage.
 258   if [[ ${filename} =~ src/main/asciidoc ]] ||
 259      [[ ${filename} =~ src/main/xslt ]]; then
 260     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 261   else
 262     # If we change our asciidoc, rebuild mvnsite
 263     if [[ ${BUILDTOOL} = maven ]]; then
 264       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 265         yetus_debug "tests/mvnsite: ${filename}"
 266         add_test mvnsite
 267       fi
 268     fi
 269     # If we change checkstyle configs, run checkstyle
 270     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 271       yetus_debug "tests/checkstyle: ${filename}"
 272       add_test checkstyle
 273     fi
 274     # fallback to checking which tests based on what yetus would do by default
 275     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 276       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 277     elif declare -f builtin_personality_file_tests >/dev/null; then
 278       builtin_personality_file_tests "${filename}"
 279     fi
 280   fi
 281 }
 282
 283 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 284 #                tests and sets given variable to arguments to be passes to maven command.
 285 ## @audience     private
 286 ## @stability    evolving
 287 ## @param        name of variable to set with maven arguments
 288 function get_include_exclude_tests_arg
 289 {
 290   local  __resultvar=$1
 291   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 292   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 293   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 294       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 295         excludes=$(cat excludes)
 296         yetus_debug "excludes=${excludes}"
 297         if [[ -n "${excludes}" ]]; then
 298           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 299         fi
 300         rm excludes
 301       else
 302         yetus_error "Wget error $? in fetching excludes file from url" \
 303              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 304       fi
 305   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 306       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 307         includes=$(cat includes)
 308         yetus_debug "includes=${includes}"
 309         if [[ -n "${includes}" ]]; then
 310           eval "${__resultvar}='-Dtest=${includes}'"
 311         fi
 312         rm includes
 313       else
 314         yetus_error "Wget error $? in fetching includes file from url" \
 315              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 316       fi
 317   else
 318     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 319     FLAKY_URL="https://builds.apache.org/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
 320     if wget "${FLAKY_URL}" -O "excludes"; then
 321       excludes=$(cat excludes)
 322         yetus_debug "excludes=${excludes}"
 323         if [[ -n "${excludes}" ]]; then
 324           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 325         fi
 326         rm excludes
 327       else
 328         yetus_error "Wget error $? in fetching excludes file from url" \
 329              "${FLAKY_URL}. Ignoring and proceeding."
 330       fi
 331   fi
 332 }
 333
 334 ###################################################
 335 # Below here are our one-off tests specific to hbase.
 336 # TODO break them into individual files so it's easier to maintain them?
 337
 338 # TODO line length check? could ignore all java files since checkstyle gets them.
 339
 340 ###################################################
 341
 342 add_test_type refguide
 343
 344 function refguide_initialize
 345 {
 346   maven_add_install refguide
 347 }
 348
 349 function refguide_filefilter
 350 {
 351   local filename=$1
 352
 353   if [[ ${filename} =~ src/main/asciidoc ]] ||
 354      [[ ${filename} =~ src/main/xslt ]] ||
 355      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default.xml ]]; then
 356     add_test refguide
 357   fi
 358 }
 359
 360 function refguide_rebuild
 361 {
 362   local repostatus=$1
 363   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 364   declare -i count
 365   declare pdf_output
 366
 367   if ! verify_needed_test refguide; then
 368     return 0
 369   fi
 370
 371   big_console_header "Checking we can create the ref guide on ${repostatus}"
 372
 373   start_clock
 374
 375   # disabled because "maven_executor" needs to return both command and args
 376   # shellcheck disable=2046
 377   echo_and_redirect "${logfile}" \
 378     $(maven_executor) clean site --batch-mode \
 379       -pl . \
 380       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 381       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 382
 383   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 384   if [[ ${count} -gt 0 ]]; then
 385     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 386     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 387     return 1
 388   fi
 389
 390   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 391     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 392     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 393     return 1
 394   fi
 395
 396   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 397     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 398     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 399     return 1
 400   fi
 401
 402   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 403     pdf_output="book.pdf"
 404   else
 405     pdf_output="apache_hbase_reference_guide.pdf"
 406   fi
 407
 408   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 409     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 410     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 411     return 1
 412   fi
 413
 414   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 415   add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 416   return 0
 417 }
 418
 419 add_test_type shadedjars
 420
 421
 422 function shadedjars_initialize
 423 {
 424   yetus_debug "initializing shaded client checks."
 425   maven_add_install shadedjars
 426 }
 427
 428 ## @description  only run the test if java changes.
 429 ## @audience     private
 430 ## @stability    evolving
 431 ## @param        filename
 432 function shadedjars_filefilter
 433 {
 434   local filename=$1
 435
 436   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 437     add_test shadedjars
 438   fi
 439 }
 440
 441 ## @description test the shaded client artifacts
 442 ## @audience private
 443 ## @stability evolving
 444 ## @param repostatus
 445 function shadedjars_rebuild
 446 {
 447   local repostatus=$1
 448   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 449
 450   if ! verify_needed_test shadedjars; then
 451     return 0
 452   fi
 453
 454   big_console_header "Checking shaded client builds on ${repostatus}"
 455
 456   start_clock
 457
 458   # disabled because "maven_executor" needs to return both command and args
 459   # shellcheck disable=2046
 460   echo_and_redirect "${logfile}" \
 461     $(maven_executor) clean verify -fae --batch-mode \
 462       -pl hbase-shaded/hbase-shaded-check-invariants -am \
 463       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 464       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 465
 466   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 467   if [[ ${count} -gt 0 ]]; then
 468     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 469     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 470     return 1
 471   fi
 472
 473   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 474   return 0
 475 }
 476
 477 ###################################################
 478
 479 add_test_type hadoopcheck
 480
 481 ## @description  hadoopcheck file filter
 482 ## @audience     private
 483 ## @stability    evolving
 484 ## @param        filename
 485 function hadoopcheck_filefilter
 486 {
 487   local filename=$1
 488
 489   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 490     add_test hadoopcheck
 491   fi
 492 }
 493
 494 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 495 ## @audience     private
 496 ## @stability    evolving
 497 function hadoopcheck_parse_args
 498 {
 499   declare i
 500
 501   for i in "$@"; do
 502     case ${i} in
 503       --quick-hadoopcheck)
 504         delete_parameter "${i}"
 505         QUICK_HADOOPCHECK=true
 506       ;;
 507     esac
 508   done
 509 }
 510
 511 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 512 ## @audience     private
 513 ## @stability    evolving
 514 function hadoopcheck_docker_support
 515 {
 516   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 517 }
 518
 519 ## @description  hadoopcheck test
 520 ## @audience     private
 521 ## @stability    evolving
 522 ## @param        repostatus
 523 function hadoopcheck_rebuild
 524 {
 525   local repostatus=$1
 526   local hadoopver
 527   local logfile
 528   local count
 529   local result=0
 530   local hbase_hadoop2_versions
 531   local hbase_hadoop3_versions
 532
 533   if [[ "${repostatus}" = branch ]]; then
 534     return 0
 535   fi
 536
 537   if ! verify_needed_test hadoopcheck; then
 538     return 0
 539   fi
 540
 541   big_console_header "Compiling against various Hadoop versions"
 542
 543   start_clock
 544
 545   # All supported Hadoop versions that we want to test the compilation with
 546   # See the Hadoop section on prereqs in the HBase Reference Guide
 547   if [[ "${PATCH_BRANCH}" = branch-1.* ]] && [[ "${PATCH_BRANCH#branch-1.}" -lt "4" ]]; then
 548     yetus_info "Setting Hadoop 2 versions to test based on before-branch-1.4 rules."
 549     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 550       hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
 551     else
 552       hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 553     fi
 554   elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 555     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 556     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 557       hbase_hadoop2_versions="2.7.7"
 558     else
 559       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 560     fi
 561   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 562     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 563     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 564       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 565     else
 566       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 567     fi
 568   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 569     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 570     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 571       hbase_hadoop2_versions="2.7.7 2.8.5"
 572     else
 573       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 574     fi
 575   else
 576     yetus_info "Setting Hadoop 2 versions to test based on branch-1.5+/branch-2.2+/master/feature branch rules."
 577     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 578       hbase_hadoop2_versions="2.8.5 2.9.2"
 579     else
 580       hbase_hadoop2_versions="2.8.5 2.9.2"
 581     fi
 582   fi
 583   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 584     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 585     hbase_hadoop3_versions=""
 586   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 587     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 588     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 589       hbase_hadoop3_versions="3.0.3 3.1.2"
 590     else
 591       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 592     fi
 593   else
 594     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
 595     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 596       hbase_hadoop3_versions="3.1.2"
 597     else
 598       hbase_hadoop3_versions="3.1.1 3.1.2"
 599     fi
 600   fi
 601
 602   export MAVEN_OPTS="${MAVEN_OPTS}"
 603   for hadoopver in ${hbase_hadoop2_versions}; do
 604     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 605     # disabled because "maven_executor" needs to return both command and args
 606     # shellcheck disable=2046
 607     echo_and_redirect "${logfile}" \
 608       $(maven_executor) clean install \
 609         -DskipTests -DHBasePatchProcess \
 610         -Dhadoop-two.version="${hadoopver}"
 611     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 612     if [[ ${count} -gt 0 ]]; then
 613       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 614       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 615       ((result=result+1))
 616     fi
 617   done
 618
 619   for hadoopver in ${hbase_hadoop3_versions}; do
 620     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 621     # disabled because "maven_executor" needs to return both command and args
 622     # shellcheck disable=2046
 623     echo_and_redirect "${logfile}" \
 624       $(maven_executor) clean install \
 625         -DskipTests -DHBasePatchProcess \
 626         -Dhadoop-three.version="${hadoopver}" \
 627         -Dhadoop.profile=3.0
 628     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 629     if [[ ${count} -gt 0 ]]; then
 630       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 631       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 632       ((result=result+1))
 633     fi
 634   done
 635
 636   if [[ ${result} -gt 0 ]]; then
 637     return 1
 638   fi
 639
 640   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 641     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 642   else
 643     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 644   fi
 645
 646   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 647   echo_and_redirect "${logfile}" \
 648     $(maven_executor) clean install \
 649       -DskipTests -DHBasePatchProcess
 650
 651   return 0
 652 }
 653
 654 ######################################
 655
 656 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 657 add_test_type hbaseprotoc
 658
 659 ## @description  hbaseprotoc file filter
 660 ## @audience     private
 661 ## @stability    evolving
 662 ## @param        filename
 663 function hbaseprotoc_filefilter
 664 {
 665   local filename=$1
 666
 667   if [[ ${filename} =~ \.proto$ ]]; then
 668     add_test hbaseprotoc
 669   fi
 670 }
 671
 672 ## @description  check hbase proto compilation
 673 ## @audience     private
 674 ## @stability    evolving
 675 ## @param        repostatus
 676 function hbaseprotoc_rebuild
 677 {
 678   declare repostatus=$1
 679   declare i=0
 680   declare fn
 681   declare module
 682   declare logfile
 683   declare count
 684   declare result
 685
 686   if [[ "${repostatus}" = branch ]]; then
 687     return 0
 688   fi
 689
 690   if ! verify_needed_test hbaseprotoc; then
 691     return 0
 692   fi
 693
 694   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 695
 696   start_clock
 697
 698   personality_modules patch hbaseprotoc
 699   # Need to run 'install' instead of 'compile' because shading plugin
 700   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 701   # half of its process done.
 702   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 703
 704   # shellcheck disable=SC2153
 705   until [[ $i -eq "${#MODULE[@]}" ]]; do
 706     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 707       ((result=result+1))
 708       ((i=i+1))
 709       continue
 710     fi
 711     module=${MODULE[$i]}
 712     fn=$(module_file_fragment "${module}")
 713     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 714
 715     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 716
 717     if [[ ${count} -gt 0 ]]; then
 718       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 719         "${count} new protoc errors in ${module}."
 720       ((result=result+1))
 721     fi
 722     ((i=i+1))
 723   done
 724
 725   modules_messages patch hbaseprotoc true
 726   if [[ ${result} -gt 0 ]]; then
 727     return 1
 728   fi
 729   return 0
 730 }
 731
 732 ######################################
 733
 734 add_test_type hbaseanti
 735
 736 ## @description  hbaseanti file filter
 737 ## @audience     private
 738 ## @stability    evolving
 739 ## @param        filename
 740 function hbaseanti_filefilter
 741 {
 742   local filename=$1
 743
 744   if [[ ${filename} =~ \.java$ ]]; then
 745     add_test hbaseanti
 746   fi
 747 }
 748
 749 ## @description  hbaseanti patch file check
 750 ## @audience     private
 751 ## @stability    evolving
 752 ## @param        filename
 753 function hbaseanti_patchfile
 754 {
 755   local patchfile=$1
 756   local warnings
 757   local result
 758
 759   if [[ "${BUILDMODE}" = full ]]; then
 760     return 0
 761   fi
 762
 763   if ! verify_needed_test hbaseanti; then
 764     return 0
 765   fi
 766
 767   big_console_header "Checking for known anti-patterns"
 768
 769   start_clock
 770
 771   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 772   if [[ ${warnings} -gt 0 ]]; then
 773     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 774     ((result=result+1))
 775   fi
 776
 777   if [[ ${result} -gt 0 ]]; then
 778     return 1
 779   fi
 780
 781   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 782   return 0
 783 }
 784
 785 ## @description  process the javac output for generating WARNING/ERROR
 786 ## @audience     private
 787 ## @stability    evolving
 788 ## @param        input filename
 789 ## @param        output filename
 790 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 791 # This is because that the output order of the error prone warnings is not stable, so the diff
 792 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 793 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 794 # really hard to sort by file name first and then line number and column number in shell...
 795 function hbase_javac_logfilter
 796 {
 797   declare input=$1
 798   declare output=$2
 799
 800   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 801 }
 802
 803 ## This is named so that yetus will check us right after running tests.
 804 ## Essentially, we check for normal failures and then we look for zombies.
 805 #function hbase_unit_logfilter
 806 #{
 807 #  declare testtype="unit"
 808 #  declare input=$1
 809 #  declare output=$2
 810 #  declare processes
 811 #  declare process_output
 812 #  declare zombies
 813 #  declare zombie_count=0
 814 #  declare zombie_process
 815 #
 816 #  yetus_debug "in hbase-specific unit logfilter."
 817 #
 818 #  # pass-through to whatever is counting actual failures
 819 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 820 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 821 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 822 #    "${testtype}_logfilter" "${input}" "${output}"
 823 #  fi
 824 #
 825 #  start_clock
 826 #  if [ -n "${BUILD_ID}" ]; then
 827 #    yetus_debug "Checking for zombie test processes."
 828 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 829 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 830 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 831 #      yetus_debug "${processes}"
 832 #      sleep 30
 833 #      #shellcheck disable=SC2016
 834 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 835 #        # Test our zombie still running (and that it still an hbase build item)
 836 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 837 #        if [[ -n "${process_output}" ]]; then
 838 #          yetus_error "Zombie: ${process_output}"
 839 #          ((zombie_count = zombie_count + 1))
 840 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 841 #          zombies="${zombies} ${zombie_process}"
 842 #        fi
 843 #      done
 844 #    fi
 845 #    if [ "${zombie_count}" -ne 0 ]; then
 846 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 847 #      populate_test_table "zombie unit tests" "${zombies}"
 848 #    else
 849 #      yetus_info "Zombie check complete. All test runs exited normally."
 850 #      stop_clock
 851 #    fi
 852 #  else
 853 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 854 #  fi
 855 #
 856 #}