dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--jenkins` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 ## @description  Globals specific to this personality
  49 ## @audience     private
  50 ## @stability    evolving
  51 function personality_globals
  52 {
  53   BUILDTOOL=maven
  54   #shellcheck disable=SC2034
  55   PROJECT_NAME=hbase
  56   #shellcheck disable=SC2034
  57   PATCH_BRANCH_DEFAULT=master
  58   #shellcheck disable=SC2034
  59   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  60   #shellcheck disable=SC2034
  61   GITHUB_REPO="apache/hbase"
  62
  63   # TODO use PATCH_BRANCH to select jdk versions to use.
  64
  65   # Override the maven options
  66   MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
  67
  68   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  69   # Up it. See HBASE-19902 for how we arrived at this number.
  70   #shellcheck disable=SC2034
  71   PROCLIMIT=10000
  72
  73   # Set docker container to run with 20g. Default is 4g in yetus.
  74   # See HBASE-19902 for how we arrived at 20g.
  75   #shellcheck disable=SC2034
  76   DOCKERMEMLIMIT=20g
  77 }
  78
  79 ## @description  Parse extra arguments required by personalities, if any.
  80 ## @audience     private
  81 ## @stability    evolving
  82 function personality_parse_args
  83 {
  84   declare i
  85
  86   for i in "$@"; do
  87     case ${i} in
  88       --exclude-tests-url=*)
  89         EXCLUDE_TESTS_URL=${i#*=}
  90       ;;
  91       --include-tests-url=*)
  92         INCLUDE_TESTS_URL=${i#*=}
  93       ;;
  94       --hadoop-profile=*)
  95         HADOOP_PROFILE=${i#*=}
  96       ;;
  97       --skip-errorprone)
  98         SKIP_ERRORPRONE=true
  99       ;;
 100     esac
 101   done
 102 }
 103
 104 ## @description  Queue up modules for this personality
 105 ## @audience     private
 106 ## @stability    evolving
 107 ## @param        repostatus
 108 ## @param        testtype
 109 function personality_modules
 110 {
 111   local repostatus=$1
 112   local testtype=$2
 113   local extra=""
 114   local MODULES=("${CHANGED_MODULES[@]}")
 115
 116   yetus_info "Personality: ${repostatus} ${testtype}"
 117
 118   clear_personality_queue
 119
 120   extra="-DHBasePatchProcess"
 121   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 122     extra="${extra} -Dhttps.protocols=TLSv1.2"
 123   fi
 124
 125   if [[ -n "${HADOOP_PROFILE}" ]]; then
 126     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 127   fi
 128
 129   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 130   # full source code instead. In this case, do full compiles, tests, etc instead of per
 131   # module.
 132   # Used in nightly runs.
 133   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 134   # modules if root is included. HBASE-18505
 135   if [[ "${BUILDMODE}" == "full" ]] || \
 136      ( ( [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]] ) && \
 137      [[ "${MODULES[*]}" =~ \. ]] ); then
 138     MODULES=(.)
 139   fi
 140
 141   # If the checkstyle configs change, check everything.
 142   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 143     MODULES=(.)
 144   fi
 145
 146   if [[ ${testtype} == mvninstall ]]; then
 147     # shellcheck disable=SC2086
 148     personality_enqueue_module . ${extra}
 149     return
 150   fi
 151
 152   if [[ ${testtype} == findbugs ]]; then
 153     # Run findbugs on each module individually to diff pre-patch and post-patch results and
 154     # report new warnings for changed modules only.
 155     # For some reason, findbugs on root is not working, but running on individual modules is
 156     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 157     for module in "${CHANGED_MODULES[@]}"; do
 158       # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
 159       # in src/main/java where findbugs goes to look
 160       if [[ ${module} == hbase-shell ]]; then
 161         continue
 162       elif [[ ${module} == hbase-it ]]; then
 163         continue
 164       else
 165         # shellcheck disable=SC2086
 166         personality_enqueue_module ${module} ${extra}
 167       fi
 168     done
 169     return
 170   fi
 171
 172   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]]; then
 173     extra="${extra} -PerrorProne"
 174   fi
 175
 176   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 177   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 178   # tests respectively.
 179   if [[ ${testtype} == unit ]]; then
 180     local tests_arg=""
 181     get_include_exclude_tests_arg tests_arg
 182     extra="${extra} -PrunAllTests ${tests_arg}"
 183
 184     # Inject the jenkins build-id for our surefire invocations
 185     # Used by zombie detection stuff, even though we're not including that yet.
 186     if [ -n "${BUILD_ID}" ]; then
 187       extra="${extra} -Dbuild.id=${BUILD_ID}"
 188     fi
 189
 190     # If the set of changed files includes CommonFSUtils then add the hbase-server
 191     # module to the set of modules (if not already included) to be tested
 192     for f in "${CHANGED_FILES[@]}"
 193     do
 194       if [[ "${f}" =~ CommonFSUtils ]]; then
 195         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 196           MODULES+=("hbase-server")
 197         fi
 198         break
 199       fi
 200     done
 201   fi
 202
 203   for module in "${MODULES[@]}"; do
 204     # shellcheck disable=SC2086
 205     personality_enqueue_module ${module} ${extra}
 206   done
 207 }
 208
 209 ## @description places where we override the built in assumptions about what tests to run
 210 ## @audience    private
 211 ## @stability   evolving
 212 ## @param       filename of changed file
 213 function personality_file_tests
 214 {
 215   local filename=$1
 216   yetus_debug "HBase specific personality_file_tests"
 217   # If the change is to the refguide, then we don't need any builtin yetus tests
 218   # the refguide test (below) will suffice for coverage.
 219   if [[ ${filename} =~ src/main/asciidoc ]] ||
 220      [[ ${filename} =~ src/main/xslt ]]; then
 221     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 222   else
 223     # If we change our asciidoc, rebuild mvnsite
 224     if [[ ${BUILDTOOL} = maven ]]; then
 225       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 226         yetus_debug "tests/mvnsite: ${filename}"
 227         add_test mvnsite
 228       fi
 229     fi
 230     # If we change checkstyle configs, run checkstyle
 231     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 232       yetus_debug "tests/checkstyle: ${filename}"
 233       add_test checkstyle
 234     fi
 235     # fallback to checking which tests based on what yetus would do by default
 236     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 237       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 238     elif declare -f builtin_personality_file_tests >/dev/null; then
 239       builtin_personality_file_tests "${filename}"
 240     fi
 241   fi
 242 }
 243
 244 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 245 #                tests and sets given variable to arguments to be passes to maven command.
 246 ## @audience     private
 247 ## @stability    evolving
 248 ## @param        name of variable to set with maven arguments
 249 function get_include_exclude_tests_arg
 250 {
 251   local  __resultvar=$1
 252   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 253   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 254   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 255       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 256         excludes=$(cat excludes)
 257         yetus_debug "excludes=${excludes}"
 258         if [[ -n "${excludes}" ]]; then
 259           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 260         fi
 261         rm excludes
 262       else
 263         yetus_error "Wget error $? in fetching excludes file from url" \
 264              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 265       fi
 266   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 267       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 268         includes=$(cat includes)
 269         yetus_debug "includes=${includes}"
 270         if [[ -n "${includes}" ]]; then
 271           eval "${__resultvar}='-Dtest=${includes}'"
 272         fi
 273         rm includes
 274       else
 275         yetus_error "Wget error $? in fetching includes file from url" \
 276              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 277       fi
 278   else
 279     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 280     FLAKY_URL="https://builds.apache.org/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
 281     if wget "${FLAKY_URL}" -O "excludes"; then
 282       excludes=$(cat excludes)
 283         yetus_debug "excludes=${excludes}"
 284         if [[ -n "${excludes}" ]]; then
 285           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 286         fi
 287         rm excludes
 288       else
 289         yetus_error "Wget error $? in fetching excludes file from url" \
 290              "${FLAKY_URL}. Ignoring and proceeding."
 291       fi
 292   fi
 293 }
 294
 295 ###################################################
 296 # Below here are our one-off tests specific to hbase.
 297 # TODO break them into individual files so it's easier to maintain them?
 298
 299 # TODO line length check? could ignore all java files since checkstyle gets them.
 300
 301 ###################################################
 302
 303 add_test_type refguide
 304
 305 function refguide_initialize
 306 {
 307   maven_add_install refguide
 308 }
 309
 310 function refguide_filefilter
 311 {
 312   local filename=$1
 313
 314   if [[ ${filename} =~ src/main/asciidoc ]] ||
 315      [[ ${filename} =~ src/main/xslt ]] ||
 316      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default.xml ]]; then
 317     add_test refguide
 318   fi
 319 }
 320
 321 function refguide_rebuild
 322 {
 323   local repostatus=$1
 324   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 325   declare -i count
 326   declare pdf_output
 327
 328   if ! verify_needed_test refguide; then
 329     return 0
 330   fi
 331
 332   big_console_header "Checking we can create the ref guide on ${repostatus}"
 333
 334   start_clock
 335
 336   # disabled because "maven_executor" needs to return both command and args
 337   # shellcheck disable=2046
 338   echo_and_redirect "${logfile}" \
 339     $(maven_executor) clean site --batch-mode \
 340       -pl . \
 341       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 342       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 343
 344   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 345   if [[ ${count} -gt 0 ]]; then
 346     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 347     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 348     return 1
 349   fi
 350
 351   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 352     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 353     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 354     return 1
 355   fi
 356
 357   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 358     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 359     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 360     return 1
 361   fi
 362
 363   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 364     pdf_output="book.pdf"
 365   else
 366     pdf_output="apache_hbase_reference_guide.pdf"
 367   fi
 368
 369   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 370     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 371     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 372     return 1
 373   fi
 374
 375   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 376   add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 377   return 0
 378 }
 379
 380 add_test_type shadedjars
 381
 382
 383 function shadedjars_initialize
 384 {
 385   yetus_debug "initializing shaded client checks."
 386   maven_add_install shadedjars
 387 }
 388
 389 ## @description  only run the test if java changes.
 390 ## @audience     private
 391 ## @stability    evolving
 392 ## @param        filename
 393 function shadedjars_filefilter
 394 {
 395   local filename=$1
 396
 397   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 398     add_test shadedjars
 399   fi
 400 }
 401
 402 ## @description test the shaded client artifacts
 403 ## @audience private
 404 ## @stability evolving
 405 ## @param repostatus
 406 function shadedjars_rebuild
 407 {
 408   local repostatus=$1
 409   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 410
 411   if ! verify_needed_test shadedjars; then
 412     return 0
 413   fi
 414
 415   big_console_header "Checking shaded client builds on ${repostatus}"
 416
 417   start_clock
 418
 419   # disabled because "maven_executor" needs to return both command and args
 420   # shellcheck disable=2046
 421   echo_and_redirect "${logfile}" \
 422     $(maven_executor) clean verify -fae --batch-mode \
 423       -pl hbase-shaded/hbase-shaded-check-invariants -am \
 424       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 425       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 426
 427   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 428   if [[ ${count} -gt 0 ]]; then
 429     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 430     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 431     return 1
 432   fi
 433
 434   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 435   return 0
 436 }
 437
 438 ###################################################
 439
 440 add_test_type hadoopcheck
 441
 442 ## @description  hadoopcheck file filter
 443 ## @audience     private
 444 ## @stability    evolving
 445 ## @param        filename
 446 function hadoopcheck_filefilter
 447 {
 448   local filename=$1
 449
 450   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 451     add_test hadoopcheck
 452   fi
 453 }
 454
 455 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 456 ## @audience     private
 457 ## @stability    evolving
 458 function hadoopcheck_parse_args
 459 {
 460   declare i
 461
 462   for i in "$@"; do
 463     case ${i} in
 464       --quick-hadoopcheck)
 465         QUICK_HADOOPCHECK=true
 466       ;;
 467     esac
 468   done
 469 }
 470
 471 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 472 ## @audience     private
 473 ## @stability    evolving
 474 function hadoopcheck_docker_support
 475 {
 476   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 477 }
 478
 479 ## @description  hadoopcheck test
 480 ## @audience     private
 481 ## @stability    evolving
 482 ## @param        repostatus
 483 function hadoopcheck_rebuild
 484 {
 485   local repostatus=$1
 486   local hadoopver
 487   local logfile
 488   local count
 489   local result=0
 490   local hbase_hadoop2_versions
 491   local hbase_hadoop3_versions
 492
 493   if [[ "${repostatus}" = branch ]]; then
 494     return 0
 495   fi
 496
 497   if ! verify_needed_test hadoopcheck; then
 498     return 0
 499   fi
 500
 501   big_console_header "Compiling against various Hadoop versions"
 502
 503   start_clock
 504
 505   # All supported Hadoop versions that we want to test the compilation with
 506   # See the Hadoop section on prereqs in the HBase Reference Guide
 507   if [[ "${PATCH_BRANCH}" = branch-1.* ]] && [[ "${PATCH_BRANCH#branch-1.}" -lt "4" ]]; then
 508     yetus_info "Setting Hadoop 2 versions to test based on before-branch-1.4 rules."
 509     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 510       hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
 511     else
 512       hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 513     fi
 514   elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 515     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 516     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 517       hbase_hadoop2_versions="2.7.7"
 518     else
 519       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 520     fi
 521   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 522     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 523     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 524       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 525     else
 526       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 527     fi
 528   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 529     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 530     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 531       hbase_hadoop2_versions="2.7.7 2.8.5"
 532     else
 533       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 534     fi
 535   else
 536     yetus_info "Setting Hadoop 2 versions to test based on branch-1.5+/branch-2.2+/master/feature branch rules."
 537     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 538       hbase_hadoop2_versions="2.8.5 2.9.2"
 539     else
 540       hbase_hadoop2_versions="2.8.5 2.9.2"
 541     fi
 542   fi
 543   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 544     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 545     hbase_hadoop3_versions=""
 546   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 547     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 548     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 549       hbase_hadoop3_versions="3.0.3 3.1.2"
 550     else
 551       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 552     fi
 553   else
 554     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
 555     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 556       hbase_hadoop3_versions="3.1.2"
 557     else
 558       hbase_hadoop3_versions="3.1.1 3.1.2"
 559     fi
 560   fi
 561
 562   export MAVEN_OPTS="${MAVEN_OPTS}"
 563   for hadoopver in ${hbase_hadoop2_versions}; do
 564     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 565     # disabled because "maven_executor" needs to return both command and args
 566     # shellcheck disable=2046
 567     echo_and_redirect "${logfile}" \
 568       $(maven_executor) clean install \
 569         -DskipTests -DHBasePatchProcess \
 570         -Dhadoop-two.version="${hadoopver}"
 571     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 572     if [[ ${count} -gt 0 ]]; then
 573       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 574       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 575       ((result=result+1))
 576     fi
 577   done
 578
 579   for hadoopver in ${hbase_hadoop3_versions}; do
 580     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 581     # disabled because "maven_executor" needs to return both command and args
 582     # shellcheck disable=2046
 583     echo_and_redirect "${logfile}" \
 584       $(maven_executor) clean install \
 585         -DskipTests -DHBasePatchProcess \
 586         -Dhadoop-three.version="${hadoopver}" \
 587         -Dhadoop.profile=3.0
 588     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 589     if [[ ${count} -gt 0 ]]; then
 590       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 591       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 592       ((result=result+1))
 593     fi
 594   done
 595
 596   if [[ ${result} -gt 0 ]]; then
 597     return 1
 598   fi
 599
 600   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 601     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 602   else
 603     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 604   fi
 605
 606   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 607   echo_and_redirect "${logfile}" \
 608     $(maven_executor) clean install \
 609       -DskipTests -DHBasePatchProcess
 610
 611   return 0
 612 }
 613
 614 ######################################
 615
 616 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 617 add_test_type hbaseprotoc
 618
 619 ## @description  hbaseprotoc file filter
 620 ## @audience     private
 621 ## @stability    evolving
 622 ## @param        filename
 623 function hbaseprotoc_filefilter
 624 {
 625   local filename=$1
 626
 627   if [[ ${filename} =~ \.proto$ ]]; then
 628     add_test hbaseprotoc
 629   fi
 630 }
 631
 632 ## @description  check hbase proto compilation
 633 ## @audience     private
 634 ## @stability    evolving
 635 ## @param        repostatus
 636 function hbaseprotoc_rebuild
 637 {
 638   declare repostatus=$1
 639   declare i=0
 640   declare fn
 641   declare module
 642   declare logfile
 643   declare count
 644   declare result
 645
 646   if [[ "${repostatus}" = branch ]]; then
 647     return 0
 648   fi
 649
 650   if ! verify_needed_test hbaseprotoc; then
 651     return 0
 652   fi
 653
 654   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 655
 656   start_clock
 657
 658   personality_modules patch hbaseprotoc
 659   # Need to run 'install' instead of 'compile' because shading plugin
 660   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 661   # half of its process done.
 662   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 663
 664   # shellcheck disable=SC2153
 665   until [[ $i -eq "${#MODULE[@]}" ]]; do
 666     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 667       ((result=result+1))
 668       ((i=i+1))
 669       continue
 670     fi
 671     module=${MODULE[$i]}
 672     fn=$(module_file_fragment "${module}")
 673     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 674
 675     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 676
 677     if [[ ${count} -gt 0 ]]; then
 678       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 679         "${count} new protoc errors in ${module}."
 680       ((result=result+1))
 681     fi
 682     ((i=i+1))
 683   done
 684
 685   modules_messages patch hbaseprotoc true
 686   if [[ ${result} -gt 0 ]]; then
 687     return 1
 688   fi
 689   return 0
 690 }
 691
 692 ######################################
 693
 694 add_test_type hbaseanti
 695
 696 ## @description  hbaseanti file filter
 697 ## @audience     private
 698 ## @stability    evolving
 699 ## @param        filename
 700 function hbaseanti_filefilter
 701 {
 702   local filename=$1
 703
 704   if [[ ${filename} =~ \.java$ ]]; then
 705     add_test hbaseanti
 706   fi
 707 }
 708
 709 ## @description  hbaseanti patch file check
 710 ## @audience     private
 711 ## @stability    evolving
 712 ## @param        filename
 713 function hbaseanti_patchfile
 714 {
 715   local patchfile=$1
 716   local warnings
 717   local result
 718
 719   if [[ "${BUILDMODE}" = full ]]; then
 720     return 0
 721   fi
 722
 723   if ! verify_needed_test hbaseanti; then
 724     return 0
 725   fi
 726
 727   big_console_header "Checking for known anti-patterns"
 728
 729   start_clock
 730
 731   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 732   if [[ ${warnings} -gt 0 ]]; then
 733     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 734     ((result=result+1))
 735   fi
 736
 737   if [[ ${result} -gt 0 ]]; then
 738     return 1
 739   fi
 740
 741   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 742   return 0
 743 }
 744
 745 ## @description  process the javac output for generating WARNING/ERROR
 746 ## @audience     private
 747 ## @stability    evolving
 748 ## @param        input filename
 749 ## @param        output filename
 750 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 751 # This is because that the output order of the error prone warnings is not stable, so the diff
 752 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 753 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 754 # really hard to sort by file name first and then line number and column number in shell...
 755 function hbase_javac_logfilter
 756 {
 757   declare input=$1
 758   declare output=$2
 759
 760   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 761 }
 762
 763 ## This is named so that yetus will check us right after running tests.
 764 ## Essentially, we check for normal failures and then we look for zombies.
 765 #function hbase_unit_logfilter
 766 #{
 767 #  declare testtype="unit"
 768 #  declare input=$1
 769 #  declare output=$2
 770 #  declare processes
 771 #  declare process_output
 772 #  declare zombies
 773 #  declare zombie_count=0
 774 #  declare zombie_process
 775 #
 776 #  yetus_debug "in hbase-specific unit logfilter."
 777 #
 778 #  # pass-through to whatever is counting actual failures
 779 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 780 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 781 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 782 #    "${testtype}_logfilter" "${input}" "${output}"
 783 #  fi
 784 #
 785 #  start_clock
 786 #  if [ -n "${BUILD_ID}" ]; then
 787 #    yetus_debug "Checking for zombie test processes."
 788 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 789 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 790 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 791 #      yetus_debug "${processes}"
 792 #      sleep 30
 793 #      #shellcheck disable=SC2016
 794 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 795 #        # Test our zombie still running (and that it still an hbase build item)
 796 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 797 #        if [[ -n "${process_output}" ]]; then
 798 #          yetus_error "Zombie: ${process_output}"
 799 #          ((zombie_count = zombie_count + 1))
 800 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 801 #          zombies="${zombies} ${zombie_process}"
 802 #        fi
 803 #      done
 804 #    fi
 805 #    if [ "${zombie_count}" -ne 0 ]; then
 806 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 807 #      populate_test_table "zombie unit tests" "${zombies}"
 808 #    else
 809 #      yetus_info "Zombie check complete. All test runs exited normally."
 810 #      stop_clock
 811 #    fi
 812 #  else
 813 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 814 #  fi
 815 #
 816 #}