dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-19902 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=12500
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118     esac
 119   done
 120 }
 121
 122 ## @description  Queue up modules for this personality
 123 ## @audience     private
 124 ## @stability    evolving
 125 ## @param        repostatus
 126 ## @param        testtype
 127 function personality_modules
 128 {
 129   local repostatus=$1
 130   local testtype=$2
 131   local extra=""
 132   local branch1jdk8=()
 133   local jdk8module=""
 134   local MODULES=("${CHANGED_MODULES[@]}")
 135
 136   yetus_info "Personality: ${repostatus} ${testtype}"
 137
 138   clear_personality_queue
 139
 140   # At a few points, hbase modules can run build, test, etc. in parallel
 141   # Let it happen. Means we'll use more CPU but should be for short bursts.
 142   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 143   extra="--threads=2 -DHBasePatchProcess"
 144   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 145     extra="${extra} -Dhttps.protocols=TLSv1.2"
 146   fi
 147
 148   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 149   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 150   # logic is not both activated within Maven.
 151   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 152     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 153   fi
 154
 155   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 156   # full source code instead. In this case, do full compiles, tests, etc instead of per
 157   # module.
 158   # Used in nightly runs.
 159   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 160   # modules if root is included. HBASE-18505
 161   if [[ "${BUILDMODE}" == "full" ]] || \
 162      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 163      [[ "${MODULES[*]}" =~ \. ]]; }; then
 164     MODULES=(.)
 165   fi
 166
 167   # If the checkstyle configs change, check everything.
 168   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 169     MODULES=(.)
 170   fi
 171
 172   if [[ ${testtype} == mvninstall ]]; then
 173     # shellcheck disable=SC2086
 174     personality_enqueue_module . ${extra}
 175     return
 176   fi
 177
 178   # This list should include any modules that require jdk8. Maven should be configured to only
 179   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 180   # module to build as yetus does if something changes in the module.  Rather than try to
 181   # figure out what jdk is in use so we can duplicate the module activation logic, just
 182   # build at the top level if anything changes in one of these modules and let maven sort it out.
 183   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 184   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 185     for jdk8module in "${branch1jdk8[@]}"; do
 186       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 187         MODULES=(.)
 188         break
 189       fi
 190     done
 191   fi
 192
 193   if [[ ${testtype} == spotbugs ]]; then
 194     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 195     # report new warnings for changed modules only.
 196     # For some reason, spotbugs on root is not working, but running on individual modules is
 197     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 198     for module in "${CHANGED_MODULES[@]}"; do
 199       # skip spotbugs on any module that lacks content in `src/main/java`
 200       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 201           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 202         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 203         continue
 204       else
 205         # shellcheck disable=SC2086
 206         personality_enqueue_module ${module} ${extra}
 207       fi
 208     done
 209     return
 210   fi
 211
 212   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 213       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 214     extra="${extra} -PerrorProne"
 215   fi
 216
 217   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 218   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 219   # tests respectively.
 220   if [[ ${testtype} == unit ]]; then
 221     local tests_arg=""
 222     get_include_exclude_tests_arg tests_arg
 223     extra="${extra} -PrunAllTests ${tests_arg}"
 224
 225     # Inject the jenkins build-id for our surefire invocations
 226     # Used by zombie detection stuff, even though we're not including that yet.
 227     if [ -n "${BUILD_ID}" ]; then
 228       extra="${extra} -Dbuild.id=${BUILD_ID}"
 229     fi
 230
 231     # If the set of changed files includes CommonFSUtils then add the hbase-server
 232     # module to the set of modules (if not already included) to be tested
 233     for f in "${CHANGED_FILES[@]}"
 234     do
 235       if [[ "${f}" =~ CommonFSUtils ]]; then
 236         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 237           MODULES+=("hbase-server")
 238         fi
 239         break
 240       fi
 241     done
 242   fi
 243
 244   for module in "${MODULES[@]}"; do
 245     # shellcheck disable=SC2086
 246     personality_enqueue_module ${module} ${extra}
 247   done
 248 }
 249
 250 ## @description places where we override the built in assumptions about what tests to run
 251 ## @audience    private
 252 ## @stability   evolving
 253 ## @param       filename of changed file
 254 function personality_file_tests
 255 {
 256   local filename=$1
 257   yetus_debug "HBase specific personality_file_tests"
 258   # If the change is to the refguide, then we don't need any builtin yetus tests
 259   # the refguide test (below) will suffice for coverage.
 260   if [[ ${filename} =~ src/main/asciidoc ]] ||
 261      [[ ${filename} =~ src/main/xslt ]]; then
 262     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 263   else
 264     # If we change our asciidoc, rebuild mvnsite
 265     if [[ ${BUILDTOOL} = maven ]]; then
 266       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 267         yetus_debug "tests/mvnsite: ${filename}"
 268         add_test mvnsite
 269       fi
 270     fi
 271     # If we change checkstyle configs, run checkstyle
 272     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 273       yetus_debug "tests/checkstyle: ${filename}"
 274       add_test checkstyle
 275     fi
 276     # fallback to checking which tests based on what yetus would do by default
 277     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 278       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 279     elif declare -f builtin_personality_file_tests >/dev/null; then
 280       builtin_personality_file_tests "${filename}"
 281     fi
 282   fi
 283 }
 284
 285 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 286 #                tests and sets given variable to arguments to be passes to maven command.
 287 ## @audience     private
 288 ## @stability    evolving
 289 ## @param        name of variable to set with maven arguments
 290 function get_include_exclude_tests_arg
 291 {
 292   local  __resultvar=$1
 293   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 294   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 295   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 296       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 297         excludes=$(cat excludes)
 298         yetus_debug "excludes=${excludes}"
 299         if [[ -n "${excludes}" ]]; then
 300           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 301         fi
 302         rm excludes
 303       else
 304         yetus_error "Wget error $? in fetching excludes file from url" \
 305              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 306       fi
 307   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 308       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 309         includes=$(cat includes)
 310         yetus_debug "includes=${includes}"
 311         if [[ -n "${includes}" ]]; then
 312           eval "${__resultvar}='-Dtest=${includes}'"
 313         fi
 314         rm includes
 315       else
 316         yetus_error "Wget error $? in fetching includes file from url" \
 317              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 318       fi
 319   else
 320     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 321     FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/output/excludes"
 322     if wget "${FLAKY_URL}" -O "excludes"; then
 323       excludes=$(cat excludes)
 324         yetus_debug "excludes=${excludes}"
 325         if [[ -n "${excludes}" ]]; then
 326           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 327         fi
 328         rm excludes
 329       else
 330         yetus_error "Wget error $? in fetching excludes file from url" \
 331              "${FLAKY_URL}. Ignoring and proceeding."
 332       fi
 333   fi
 334 }
 335
 336 ###################################################
 337 # Below here are our one-off tests specific to hbase.
 338 # TODO break them into individual files so it's easier to maintain them?
 339
 340 # TODO line length check? could ignore all java files since checkstyle gets them.
 341
 342 ###################################################
 343
 344 add_test_type refguide
 345
 346 function refguide_initialize
 347 {
 348   maven_add_install refguide
 349 }
 350
 351 function refguide_filefilter
 352 {
 353   local filename=$1
 354
 355   if [[ ${filename} =~ src/main/asciidoc ]] ||
 356      [[ ${filename} =~ src/main/xslt ]] ||
 357      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default\.xml ]]; then
 358     add_test refguide
 359   fi
 360 }
 361
 362 function refguide_rebuild
 363 {
 364   local repostatus=$1
 365   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 366   declare -i count
 367   declare pdf_output
 368
 369   if ! verify_needed_test refguide; then
 370     return 0
 371   fi
 372
 373   big_console_header "Checking we can create the ref guide on ${repostatus}"
 374
 375   start_clock
 376
 377   # disabled because "maven_executor" needs to return both command and args
 378   # shellcheck disable=2046
 379   echo_and_redirect "${logfile}" \
 380     $(maven_executor) clean site --batch-mode \
 381       -pl . \
 382       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 383       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 384
 385   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 386   if [[ ${count} -gt 0 ]]; then
 387     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 388     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 389     return 1
 390   fi
 391
 392   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 393     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 394     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 395     return 1
 396   fi
 397
 398   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 399     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 400     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 401     return 1
 402   fi
 403
 404   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 405     pdf_output="book.pdf"
 406   else
 407     pdf_output="apache_hbase_reference_guide.pdf"
 408   fi
 409
 410   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 411     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 412     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 413     return 1
 414   fi
 415
 416   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 417   add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 418   return 0
 419 }
 420
 421 add_test_type shadedjars
 422
 423
 424 function shadedjars_initialize
 425 {
 426   yetus_debug "initializing shaded client checks."
 427   maven_add_install shadedjars
 428 }
 429
 430 ## @description  only run the test if java changes.
 431 ## @audience     private
 432 ## @stability    evolving
 433 ## @param        filename
 434 function shadedjars_filefilter
 435 {
 436   local filename=$1
 437
 438   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 439     add_test shadedjars
 440   fi
 441 }
 442
 443 ## @description test the shaded client artifacts
 444 ## @audience private
 445 ## @stability evolving
 446 ## @param repostatus
 447 function shadedjars_rebuild
 448 {
 449   local repostatus=$1
 450   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 451
 452   if ! verify_needed_test shadedjars; then
 453     return 0
 454   fi
 455
 456   big_console_header "Checking shaded client builds on ${repostatus}"
 457
 458   start_clock
 459
 460   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 461     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 462     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 463     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 464   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 465   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 466   # logic is not both activated within Maven.
 467   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 468     maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
 469   fi
 470
 471   # disabled because "maven_executor" needs to return both command and args
 472   # shellcheck disable=2046
 473   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 474
 475   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 476   if [[ ${count} -gt 0 ]]; then
 477     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 478     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 479     return 1
 480   fi
 481
 482   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 483   return 0
 484 }
 485
 486 ###################################################
 487
 488 add_test_type hadoopcheck
 489
 490 ## @description  hadoopcheck file filter
 491 ## @audience     private
 492 ## @stability    evolving
 493 ## @param        filename
 494 function hadoopcheck_filefilter
 495 {
 496   local filename=$1
 497
 498   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom\.xml$ ]]; then
 499     add_test hadoopcheck
 500   fi
 501 }
 502
 503 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 504 ## @audience     private
 505 ## @stability    evolving
 506 function hadoopcheck_parse_args
 507 {
 508   declare i
 509
 510   for i in "$@"; do
 511     case ${i} in
 512       --quick-hadoopcheck)
 513         delete_parameter "${i}"
 514         QUICK_HADOOPCHECK=true
 515       ;;
 516     esac
 517   done
 518 }
 519
 520 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 521 ## @audience     private
 522 ## @stability    evolving
 523 function hadoopcheck_docker_support
 524 {
 525   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 526 }
 527
 528 ## @description  hadoopcheck test
 529 ## @audience     private
 530 ## @stability    evolving
 531 ## @param        repostatus
 532 function hadoopcheck_rebuild
 533 {
 534   local repostatus=$1
 535   local hadoopver
 536   local logfile
 537   local count
 538   local result=0
 539   local hbase_hadoop2_versions
 540   local hbase_hadoop3_versions
 541
 542   if [[ "${repostatus}" = branch ]]; then
 543     return 0
 544   fi
 545
 546   if ! verify_needed_test hadoopcheck; then
 547     return 0
 548   fi
 549
 550   big_console_header "Compiling against various Hadoop versions"
 551
 552   start_clock
 553
 554   # All supported Hadoop versions that we want to test the compilation with
 555   # See the Hadoop section on prereqs in the HBase Reference Guide
 556   if [[ "${PATCH_BRANCH}" = branch-1.3 ]]; then
 557     yetus_info "Setting Hadoop 2 versions to test based on branch-1.3 rules."
 558     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 559       hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
 560     else
 561       hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 562     fi
 563   elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 564     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 565     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 566       hbase_hadoop2_versions="2.7.7"
 567     else
 568       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 569     fi
 570   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 571     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 572     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 573       hbase_hadoop2_versions="2.10.0"
 574     else
 575       hbase_hadoop2_versions="2.10.0"
 576     fi
 577   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 578     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 579     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 580       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 581     else
 582       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 583     fi
 584   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 585     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 586     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 587       hbase_hadoop2_versions="2.7.7 2.8.5"
 588     else
 589       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 590     fi
 591   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 592     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 593     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 594       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 595     else
 596       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 597     fi
 598   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 599     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 600     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 601       hbase_hadoop2_versions="2.10.0"
 602     else
 603       hbase_hadoop2_versions="2.10.0"
 604     fi
 605   else
 606     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 607     hbase_hadoop2_versions=""
 608   fi
 609   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 610     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 611     hbase_hadoop3_versions=""
 612   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 613     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 614     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 615       hbase_hadoop3_versions="3.0.3 3.1.2"
 616     else
 617       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 618     fi
 619   else
 620     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
 621     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 622       hbase_hadoop3_versions="3.1.2 3.2.1"
 623     else
 624       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1"
 625     fi
 626   fi
 627
 628   export MAVEN_OPTS="${MAVEN_OPTS}"
 629   for hadoopver in ${hbase_hadoop2_versions}; do
 630     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 631     # disabled because "maven_executor" needs to return both command and args
 632     # shellcheck disable=2046
 633     echo_and_redirect "${logfile}" \
 634       $(maven_executor) clean install \
 635         -DskipTests -DHBasePatchProcess \
 636         -Dhadoop-two.version="${hadoopver}"
 637     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 638     if [[ ${count} -gt 0 ]]; then
 639       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 640       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 641       ((result=result+1))
 642     fi
 643   done
 644
 645   hadoop_profile=""
 646   if [[ "${PATCH_BRANCH}" = branch-2* ]]; then
 647     hadoop_profile="-Dhadoop.profile=3.0"
 648   fi
 649   for hadoopver in ${hbase_hadoop3_versions}; do
 650     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 651     # disabled because "maven_executor" needs to return both command and args
 652     # shellcheck disable=2046
 653     echo_and_redirect "${logfile}" \
 654       $(maven_executor) clean install \
 655         -DskipTests -DHBasePatchProcess \
 656         -Dhadoop-three.version="${hadoopver}" \
 657         ${hadoop_profile}
 658     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 659     if [[ ${count} -gt 0 ]]; then
 660       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 661       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 662       ((result=result+1))
 663     fi
 664   done
 665
 666   if [[ ${result} -gt 0 ]]; then
 667     return 1
 668   fi
 669
 670   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 671     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 672       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 673     else
 674       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 675     fi
 676   else
 677     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 678   fi
 679
 680   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 681   echo_and_redirect "${logfile}" \
 682     $(maven_executor) clean install \
 683       -DskipTests -DHBasePatchProcess
 684
 685   return 0
 686 }
 687
 688 ######################################
 689
 690 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 691 add_test_type hbaseprotoc
 692
 693 ## @description  hbaseprotoc file filter
 694 ## @audience     private
 695 ## @stability    evolving
 696 ## @param        filename
 697 function hbaseprotoc_filefilter
 698 {
 699   local filename=$1
 700
 701   if [[ ${filename} =~ \.proto$ ]]; then
 702     add_test hbaseprotoc
 703   fi
 704 }
 705
 706 ## @description  check hbase proto compilation
 707 ## @audience     private
 708 ## @stability    evolving
 709 ## @param        repostatus
 710 function hbaseprotoc_rebuild
 711 {
 712   declare repostatus=$1
 713   declare i=0
 714   declare fn
 715   declare module
 716   declare logfile
 717   declare count
 718   declare result
 719
 720   if [[ "${repostatus}" = branch ]]; then
 721     return 0
 722   fi
 723
 724   if ! verify_needed_test hbaseprotoc; then
 725     return 0
 726   fi
 727
 728   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 729
 730   start_clock
 731
 732   personality_modules patch hbaseprotoc
 733   # Need to run 'install' instead of 'compile' because shading plugin
 734   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 735   # half of its process done.
 736   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 737
 738   # shellcheck disable=SC2153
 739   until [[ $i -eq "${#MODULE[@]}" ]]; do
 740     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 741       ((result=result+1))
 742       ((i=i+1))
 743       continue
 744     fi
 745     module=${MODULE[$i]}
 746     fn=$(module_file_fragment "${module}")
 747     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 748
 749     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 750
 751     if [[ ${count} -gt 0 ]]; then
 752       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 753         "${count} new protoc errors in ${module}."
 754       ((result=result+1))
 755     fi
 756     ((i=i+1))
 757   done
 758
 759   modules_messages patch hbaseprotoc true
 760   if [[ ${result} -gt 0 ]]; then
 761     return 1
 762   fi
 763   return 0
 764 }
 765
 766 ######################################
 767
 768 add_test_type hbaseanti
 769
 770 ## @description  hbaseanti file filter
 771 ## @audience     private
 772 ## @stability    evolving
 773 ## @param        filename
 774 function hbaseanti_filefilter
 775 {
 776   local filename=$1
 777
 778   if [[ ${filename} =~ \.java$ ]]; then
 779     add_test hbaseanti
 780   fi
 781 }
 782
 783 ## @description  hbaseanti patch file check
 784 ## @audience     private
 785 ## @stability    evolving
 786 ## @param        filename
 787 function hbaseanti_patchfile
 788 {
 789   local patchfile=$1
 790   local warnings
 791   local result
 792
 793   if [[ "${BUILDMODE}" = full ]]; then
 794     return 0
 795   fi
 796
 797   if ! verify_needed_test hbaseanti; then
 798     return 0
 799   fi
 800
 801   big_console_header "Checking for known anti-patterns"
 802
 803   start_clock
 804
 805   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 806   if [[ ${warnings} -gt 0 ]]; then
 807     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 808     ((result=result+1))
 809   fi
 810
 811   if [[ ${result} -gt 0 ]]; then
 812     return 1
 813   fi
 814
 815   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 816   return 0
 817 }
 818
 819 ## @description  process the javac output for generating WARNING/ERROR
 820 ## @audience     private
 821 ## @stability    evolving
 822 ## @param        input filename
 823 ## @param        output filename
 824 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 825 # This is because that the output order of the error prone warnings is not stable, so the diff
 826 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 827 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 828 # really hard to sort by file name first and then line number and column number in shell...
 829 function hbase_javac_logfilter
 830 {
 831   declare input=$1
 832   declare output=$2
 833
 834   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 835 }
 836
 837 ## This is named so that yetus will check us right after running tests.
 838 ## Essentially, we check for normal failures and then we look for zombies.
 839 #function hbase_unit_logfilter
 840 #{
 841 #  declare testtype="unit"
 842 #  declare input=$1
 843 #  declare output=$2
 844 #  declare processes
 845 #  declare process_output
 846 #  declare zombies
 847 #  declare zombie_count=0
 848 #  declare zombie_process
 849 #
 850 #  yetus_debug "in hbase-specific unit logfilter."
 851 #
 852 #  # pass-through to whatever is counting actual failures
 853 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 854 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 855 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 856 #    "${testtype}_logfilter" "${input}" "${output}"
 857 #  fi
 858 #
 859 #  start_clock
 860 #  if [ -n "${BUILD_ID}" ]; then
 861 #    yetus_debug "Checking for zombie test processes."
 862 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 863 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 864 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 865 #      yetus_debug "${processes}"
 866 #      sleep 30
 867 #      #shellcheck disable=SC2016
 868 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 869 #        # Test our zombie still running (and that it still an hbase build item)
 870 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 871 #        if [[ -n "${process_output}" ]]; then
 872 #          yetus_error "Zombie: ${process_output}"
 873 #          ((zombie_count = zombie_count + 1))
 874 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 875 #          zombies="${zombies} ${zombie_process}"
 876 #        fi
 877 #      done
 878 #    fi
 879 #    if [ "${zombie_count}" -ne 0 ]; then
 880 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 881 #      populate_test_table "zombie unit tests" "${zombies}"
 882 #    else
 883 #      yetus_info "Zombie check complete. All test runs exited normally."
 884 #      stop_clock
 885 #    fi
 886 #  else
 887 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 888 #  fi
 889 #
 890 #}