dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-25081 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=30000
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118       --asf-nightlies-general-check-base=*)
 119         delete_parameter "${i}"
 120         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
 121       ;;
 122       --build-thread=*
 123         delete_parameter "${i}"
 124         BUILD_THREAD=${i#*=}
 125       ;;
 126       --surefire-first-part-fork-count=*
 127         delete_parameter "${i}"
 128         SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
 129       ;;
 130       --surefire-second-part-fork-count=*
 131         delete_parameter "${i}"
 132         SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
 133       ;;
 134     esac
 135   done
 136 }
 137
 138 ## @description  Queue up modules for this personality
 139 ## @audience     private
 140 ## @stability    evolving
 141 ## @param        repostatus
 142 ## @param        testtype
 143 function personality_modules
 144 {
 145   local repostatus=$1
 146   local testtype=$2
 147   local extra=""
 148   local branch1jdk8=()
 149   local jdk8module=""
 150   local MODULES=("${CHANGED_MODULES[@]}")
 151
 152   yetus_info "Personality: ${repostatus} ${testtype}"
 153
 154   clear_personality_queue
 155
 156   # At a few points, hbase modules can run build, test, etc. in parallel
 157   # Let it happen. Means we'll use more CPU but should be for short bursts.
 158   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 159   if [[ -n "${BUILD_THREAD}" ]]; then
 160     extra="--threads=${BUILD_THREAD}"
 161   else
 162     extra="--threads=2"
 163   fi
 164
 165   extra="${extra} -DHBasePatchProcess"
 166   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 167     extra="${extra} -Dhttps.protocols=TLSv1.2"
 168   fi
 169
 170   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 171   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 172   # logic is not both activated within Maven.
 173   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 174     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 175   fi
 176
 177   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 178   # full source code instead. In this case, do full compiles, tests, etc instead of per
 179   # module.
 180   # Used in nightly runs.
 181   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 182   # modules if root is included. HBASE-18505
 183   if [[ "${BUILDMODE}" == "full" ]] || \
 184      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 185      [[ "${MODULES[*]}" =~ \. ]]; }; then
 186     MODULES=(.)
 187   fi
 188
 189   # If the checkstyle configs change, check everything.
 190   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 191     MODULES=(.)
 192   fi
 193
 194   if [[ ${testtype} == mvninstall ]]; then
 195     # shellcheck disable=SC2086
 196     personality_enqueue_module . ${extra}
 197     return
 198   fi
 199
 200   # This list should include any modules that require jdk8. Maven should be configured to only
 201   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 202   # module to build as yetus does if something changes in the module.  Rather than try to
 203   # figure out what jdk is in use so we can duplicate the module activation logic, just
 204   # build at the top level if anything changes in one of these modules and let maven sort it out.
 205   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 206   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 207     for jdk8module in "${branch1jdk8[@]}"; do
 208       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 209         MODULES=(.)
 210         break
 211       fi
 212     done
 213   fi
 214
 215   if [[ ${testtype} == spotbugs ]]; then
 216     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 217     # report new warnings for changed modules only.
 218     # For some reason, spotbugs on root is not working, but running on individual modules is
 219     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 220     for module in "${CHANGED_MODULES[@]}"; do
 221       # skip spotbugs on any module that lacks content in `src/main/java`
 222       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 223           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 224         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 225         continue
 226       else
 227         # shellcheck disable=SC2086
 228         personality_enqueue_module ${module} ${extra}
 229       fi
 230     done
 231     return
 232   fi
 233
 234   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 235       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 236     extra="${extra} -PerrorProne"
 237   fi
 238
 239   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 240   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 241   # tests respectively.
 242   if [[ ${testtype} == unit ]]; then
 243     local tests_arg=""
 244     get_include_exclude_tests_arg tests_arg
 245     extra="${extra} -PrunAllTests ${tests_arg}"
 246
 247     # Inject the jenkins build-id for our surefire invocations
 248     # Used by zombie detection stuff, even though we're not including that yet.
 249     if [ -n "${BUILD_ID}" ]; then
 250       extra="${extra} -Dbuild.id=${BUILD_ID}"
 251     fi
 252
 253     # set forkCount
 254     if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
 255       extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
 256     fi
 257
 258     if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
 259       extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
 260     fi
 261
 262     # If the set of changed files includes CommonFSUtils then add the hbase-server
 263     # module to the set of modules (if not already included) to be tested
 264     for f in "${CHANGED_FILES[@]}"
 265     do
 266       if [[ "${f}" =~ CommonFSUtils ]]; then
 267         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 268           MODULES+=("hbase-server")
 269         fi
 270         break
 271       fi
 272     done
 273   fi
 274
 275   for module in "${MODULES[@]}"; do
 276     # shellcheck disable=SC2086
 277     personality_enqueue_module ${module} ${extra}
 278   done
 279 }
 280
 281 ## @description places where we override the built in assumptions about what tests to run
 282 ## @audience    private
 283 ## @stability   evolving
 284 ## @param       filename of changed file
 285 function personality_file_tests
 286 {
 287   local filename=$1
 288   yetus_debug "HBase specific personality_file_tests"
 289   # If the change is to the refguide, then we don't need any builtin yetus tests
 290   # the refguide test (below) will suffice for coverage.
 291   if [[ ${filename} =~ src/main/asciidoc ]] ||
 292      [[ ${filename} =~ src/main/xslt ]]; then
 293     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 294   else
 295     # If we change our asciidoc, rebuild mvnsite
 296     if [[ ${BUILDTOOL} = maven ]]; then
 297       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 298         yetus_debug "tests/mvnsite: ${filename}"
 299         add_test mvnsite
 300       fi
 301     fi
 302     # If we change checkstyle configs, run checkstyle
 303     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 304       yetus_debug "tests/checkstyle: ${filename}"
 305       add_test checkstyle
 306     fi
 307     # fallback to checking which tests based on what yetus would do by default
 308     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 309       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 310     elif declare -f builtin_personality_file_tests >/dev/null; then
 311       builtin_personality_file_tests "${filename}"
 312     fi
 313   fi
 314 }
 315
 316 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 317 #                tests and sets given variable to arguments to be passes to maven command.
 318 ## @audience     private
 319 ## @stability    evolving
 320 ## @param        name of variable to set with maven arguments
 321 function get_include_exclude_tests_arg
 322 {
 323   local  __resultvar=$1
 324   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 325   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 326   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 327       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 328         excludes=$(cat excludes)
 329         yetus_debug "excludes=${excludes}"
 330         if [[ -n "${excludes}" ]]; then
 331           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 332         fi
 333         rm excludes
 334       else
 335         yetus_error "Wget error $? in fetching excludes file from url" \
 336              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 337       fi
 338   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 339       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 340         includes=$(cat includes)
 341         yetus_debug "includes=${includes}"
 342         if [[ -n "${includes}" ]]; then
 343           eval "${__resultvar}='-Dtest=${includes}'"
 344         fi
 345         rm includes
 346       else
 347         yetus_error "Wget error $? in fetching includes file from url" \
 348              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 349       fi
 350   else
 351     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 352     FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/output/excludes"
 353     if wget "${FLAKY_URL}" -O "excludes"; then
 354       excludes=$(cat excludes)
 355         yetus_debug "excludes=${excludes}"
 356         if [[ -n "${excludes}" ]]; then
 357           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 358         fi
 359         rm excludes
 360       else
 361         yetus_error "Wget error $? in fetching excludes file from url" \
 362              "${FLAKY_URL}. Ignoring and proceeding."
 363       fi
 364   fi
 365 }
 366
 367 ###################################################
 368 # Below here are our one-off tests specific to hbase.
 369 # TODO break them into individual files so it's easier to maintain them?
 370
 371 # TODO line length check? could ignore all java files since checkstyle gets them.
 372
 373 ###################################################
 374
 375 add_test_type refguide
 376
 377 function refguide_initialize
 378 {
 379   maven_add_install refguide
 380 }
 381
 382 function refguide_filefilter
 383 {
 384   local filename=$1
 385
 386   if [[ ${filename} =~ src/main/asciidoc ]] ||
 387      [[ ${filename} =~ src/main/xslt ]] ||
 388      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default\.xml ]]; then
 389     add_test refguide
 390   fi
 391 }
 392
 393 function refguide_rebuild
 394 {
 395   local repostatus=$1
 396   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 397   declare -i count
 398   declare pdf_output
 399
 400   if ! verify_needed_test refguide; then
 401     return 0
 402   fi
 403
 404   big_console_header "Checking we can create the ref guide on ${repostatus}"
 405
 406   start_clock
 407
 408   # disabled because "maven_executor" needs to return both command and args
 409   # shellcheck disable=2046
 410   echo_and_redirect "${logfile}" \
 411     $(maven_executor) clean site --batch-mode \
 412       -pl . \
 413       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 414       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 415
 416   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 417   if [[ ${count} -gt 0 ]]; then
 418     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 419     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 420     return 1
 421   fi
 422
 423   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 424     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 425     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 426     return 1
 427   fi
 428
 429   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 430     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 431     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 432     return 1
 433   fi
 434
 435   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 436     pdf_output="book.pdf"
 437   else
 438     pdf_output="apache_hbase_reference_guide.pdf"
 439   fi
 440
 441   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 442     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 443     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 444     return 1
 445   fi
 446
 447   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 448   if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
 449     add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
 450   else
 451     add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 452   fi
 453   return 0
 454 }
 455
 456 add_test_type shadedjars
 457
 458
 459 function shadedjars_initialize
 460 {
 461   yetus_debug "initializing shaded client checks."
 462   maven_add_install shadedjars
 463 }
 464
 465 ## @description  only run the test if java changes.
 466 ## @audience     private
 467 ## @stability    evolving
 468 ## @param        filename
 469 function shadedjars_filefilter
 470 {
 471   local filename=$1
 472
 473   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 474     add_test shadedjars
 475   fi
 476 }
 477
 478 ## @description test the shaded client artifacts
 479 ## @audience private
 480 ## @stability evolving
 481 ## @param repostatus
 482 function shadedjars_rebuild
 483 {
 484   local repostatus=$1
 485   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 486
 487   if ! verify_needed_test shadedjars; then
 488     return 0
 489   fi
 490
 491   big_console_header "Checking shaded client builds on ${repostatus}"
 492
 493   start_clock
 494
 495   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 496     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 497     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 498     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 499   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 500   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 501   # logic is not both activated within Maven.
 502   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 503     maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
 504   fi
 505
 506   # disabled because "maven_executor" needs to return both command and args
 507   # shellcheck disable=2046
 508   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 509
 510   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 511   if [[ ${count} -gt 0 ]]; then
 512     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 513     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 514     return 1
 515   fi
 516
 517   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 518   return 0
 519 }
 520
 521 ###################################################
 522
 523 add_test_type hadoopcheck
 524
 525 ## @description  hadoopcheck file filter
 526 ## @audience     private
 527 ## @stability    evolving
 528 ## @param        filename
 529 function hadoopcheck_filefilter
 530 {
 531   local filename=$1
 532
 533   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom\.xml$ ]]; then
 534     add_test hadoopcheck
 535   fi
 536 }
 537
 538 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 539 ## @audience     private
 540 ## @stability    evolving
 541 function hadoopcheck_parse_args
 542 {
 543   declare i
 544
 545   for i in "$@"; do
 546     case ${i} in
 547       --quick-hadoopcheck)
 548         delete_parameter "${i}"
 549         QUICK_HADOOPCHECK=true
 550       ;;
 551     esac
 552   done
 553 }
 554
 555 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 556 ## @audience     private
 557 ## @stability    evolving
 558 function hadoopcheck_docker_support
 559 {
 560   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 561 }
 562
 563 ## @description  hadoopcheck test
 564 ## @audience     private
 565 ## @stability    evolving
 566 ## @param        repostatus
 567 function hadoopcheck_rebuild
 568 {
 569   local repostatus=$1
 570   local hadoopver
 571   local logfile
 572   local count
 573   local result=0
 574   local hbase_hadoop2_versions
 575   local hbase_hadoop3_versions
 576
 577   if [[ "${repostatus}" = branch ]]; then
 578     return 0
 579   fi
 580
 581   if ! verify_needed_test hadoopcheck; then
 582     return 0
 583   fi
 584
 585   big_console_header "Compiling against various Hadoop versions"
 586
 587   start_clock
 588
 589   # All supported Hadoop versions that we want to test the compilation with
 590   # See the Hadoop section on prereqs in the HBase Reference Guide
 591   if [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 592     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 593     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 594       hbase_hadoop2_versions="2.7.7"
 595     else
 596       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 597     fi
 598   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 599     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 600     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 601       hbase_hadoop2_versions="2.10.0"
 602     else
 603       hbase_hadoop2_versions="2.10.0"
 604     fi
 605   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 606     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 607     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 608       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 609     else
 610       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 611     fi
 612   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 613     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 614     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 615       hbase_hadoop2_versions="2.7.7 2.8.5"
 616     else
 617       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 618     fi
 619   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 620     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 621     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 622       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 623     else
 624       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 625     fi
 626   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 627     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 628     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 629       hbase_hadoop2_versions="2.10.1"
 630     else
 631       hbase_hadoop2_versions="2.10.0 2.10.1"
 632     fi
 633   else
 634     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 635     hbase_hadoop2_versions=""
 636   fi
 637   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 638     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 639     hbase_hadoop3_versions=""
 640   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 641     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 642     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 643       hbase_hadoop3_versions="3.0.3 3.1.2"
 644     else
 645       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 646     fi
 647   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]] || [[ "${PATCH_BRANCH}" = branch-2.3 ]]; then
 648     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2/branch-2.3 rules"
 649     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 650       hbase_hadoop3_versions="3.1.2 3.2.2"
 651     else
 652       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2"
 653     fi
 654   else
 655     yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules"
 656     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 657       hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1"
 658     else
 659       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1"
 660     fi
 661   fi
 662
 663   export MAVEN_OPTS="${MAVEN_OPTS}"
 664   for hadoopver in ${hbase_hadoop2_versions}; do
 665     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 666     # disabled because "maven_executor" needs to return both command and args
 667     # shellcheck disable=2046
 668     echo_and_redirect "${logfile}" \
 669       $(maven_executor) clean install \
 670         -DskipTests -DHBasePatchProcess \
 671         -Dhadoop-two.version="${hadoopver}"
 672     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 673     if [[ ${count} -gt 0 ]]; then
 674       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 675       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 676       ((result=result+1))
 677     fi
 678   done
 679
 680   hadoop_profile=""
 681   if [[ "${PATCH_BRANCH}" = branch-2* ]]; then
 682     hadoop_profile="-Dhadoop.profile=3.0"
 683   fi
 684   for hadoopver in ${hbase_hadoop3_versions}; do
 685     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 686     # disabled because "maven_executor" needs to return both command and args
 687     # shellcheck disable=2046
 688     echo_and_redirect "${logfile}" \
 689       $(maven_executor) clean install \
 690         -DskipTests -DHBasePatchProcess \
 691         -Dhadoop-three.version="${hadoopver}" \
 692         ${hadoop_profile}
 693     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 694     if [[ ${count} -gt 0 ]]; then
 695       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 696       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 697       ((result=result+1))
 698     fi
 699   done
 700
 701   if [[ ${result} -gt 0 ]]; then
 702     return 1
 703   fi
 704
 705   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 706     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 707       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 708     else
 709       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 710     fi
 711   else
 712     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 713   fi
 714
 715   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 716   echo_and_redirect "${logfile}" \
 717     $(maven_executor) clean install \
 718       -DskipTests -DHBasePatchProcess
 719
 720   return 0
 721 }
 722
 723 ######################################
 724
 725 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 726 add_test_type hbaseprotoc
 727
 728 function hbaseprotoc_initialize
 729 {
 730   # So long as there are inter-module dependencies on the protoc modules, we
 731   # need to run a full `mvn install` before a patch can be tested.
 732   yetus_debug "initializing HBase Protoc plugin."
 733   maven_add_install hbaseprotoc
 734 }
 735
 736 ## @description  hbaseprotoc file filter
 737 ## @audience     private
 738 ## @stability    evolving
 739 ## @param        filename
 740 function hbaseprotoc_filefilter
 741 {
 742   local filename=$1
 743
 744   if [[ ${filename} =~ \.proto$ ]]; then
 745     add_test hbaseprotoc
 746   fi
 747 }
 748
 749 ## @description  check hbase proto compilation
 750 ## @audience     private
 751 ## @stability    evolving
 752 ## @param        repostatus
 753 function hbaseprotoc_rebuild
 754 {
 755   declare repostatus=$1
 756   declare i=0
 757   declare fn
 758   declare module
 759   declare logfile
 760   declare count
 761   declare result
 762
 763   if [[ "${repostatus}" = branch ]]; then
 764     return 0
 765   fi
 766
 767   if ! verify_needed_test hbaseprotoc; then
 768     return 0
 769   fi
 770
 771   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 772
 773   start_clock
 774
 775   personality_modules patch hbaseprotoc
 776   # Need to run 'install' instead of 'compile' because shading plugin
 777   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 778   # half of its process done.
 779   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 780
 781   # shellcheck disable=SC2153
 782   until [[ $i -eq "${#MODULE[@]}" ]]; do
 783     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 784       ((result=result+1))
 785       ((i=i+1))
 786       continue
 787     fi
 788     module=${MODULE[$i]}
 789     fn=$(module_file_fragment "${module}")
 790     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 791
 792     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 793
 794     if [[ ${count} -gt 0 ]]; then
 795       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 796         "${count} new protoc errors in ${module}."
 797       ((result=result+1))
 798     fi
 799     ((i=i+1))
 800   done
 801
 802   modules_messages patch hbaseprotoc true
 803   if [[ ${result} -gt 0 ]]; then
 804     return 1
 805   fi
 806   return 0
 807 }
 808
 809 ######################################
 810
 811 add_test_type hbaseanti
 812
 813 ## @description  hbaseanti file filter
 814 ## @audience     private
 815 ## @stability    evolving
 816 ## @param        filename
 817 function hbaseanti_filefilter
 818 {
 819   local filename=$1
 820
 821   if [[ ${filename} =~ \.java$ ]]; then
 822     add_test hbaseanti
 823   fi
 824 }
 825
 826 ## @description  hbaseanti patch file check
 827 ## @audience     private
 828 ## @stability    evolving
 829 ## @param        filename
 830 function hbaseanti_patchfile
 831 {
 832   local patchfile=$1
 833   local warnings
 834   local result
 835
 836   if [[ "${BUILDMODE}" = full ]]; then
 837     return 0
 838   fi
 839
 840   if ! verify_needed_test hbaseanti; then
 841     return 0
 842   fi
 843
 844   big_console_header "Checking for known anti-patterns"
 845
 846   start_clock
 847
 848   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 849   if [[ ${warnings} -gt 0 ]]; then
 850     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 851     ((result=result+1))
 852   fi
 853
 854   if [[ ${result} -gt 0 ]]; then
 855     return 1
 856   fi
 857
 858   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 859   return 0
 860 }
 861
 862 ## @description  process the javac output for generating WARNING/ERROR
 863 ## @audience     private
 864 ## @stability    evolving
 865 ## @param        input filename
 866 ## @param        output filename
 867 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 868 # This is because that the output order of the error prone warnings is not stable, so the diff
 869 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 870 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 871 # really hard to sort by file name first and then line number and column number in shell...
 872 function hbase_javac_logfilter
 873 {
 874   declare input=$1
 875   declare output=$2
 876
 877   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 878 }
 879
 880 ## This is named so that yetus will check us right after running tests.
 881 ## Essentially, we check for normal failures and then we look for zombies.
 882 #function hbase_unit_logfilter
 883 #{
 884 #  declare testtype="unit"
 885 #  declare input=$1
 886 #  declare output=$2
 887 #  declare processes
 888 #  declare process_output
 889 #  declare zombies
 890 #  declare zombie_count=0
 891 #  declare zombie_process
 892 #
 893 #  yetus_debug "in hbase-specific unit logfilter."
 894 #
 895 #  # pass-through to whatever is counting actual failures
 896 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 897 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 898 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 899 #    "${testtype}_logfilter" "${input}" "${output}"
 900 #  fi
 901 #
 902 #  start_clock
 903 #  if [ -n "${BUILD_ID}" ]; then
 904 #    yetus_debug "Checking for zombie test processes."
 905 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 906 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 907 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 908 #      yetus_debug "${processes}"
 909 #      sleep 30
 910 #      #shellcheck disable=SC2016
 911 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 912 #        # Test our zombie still running (and that it still an hbase build item)
 913 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 914 #        if [[ -n "${process_output}" ]]; then
 915 #          yetus_error "Zombie: ${process_output}"
 916 #          ((zombie_count = zombie_count + 1))
 917 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 918 #          zombies="${zombies} ${zombie_process}"
 919 #        fi
 920 #      done
 921 #    fi
 922 #    if [ "${zombie_count}" -ne 0 ]; then
 923 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 924 #      populate_test_table "zombie unit tests" "${zombies}"
 925 #    else
 926 #      yetus_info "Zombie check complete. All test runs exited normally."
 927 #      stop_clock
 928 #    fi
 929 #  else
 930 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 931 #  fi
 932 #
 933 #}