dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-25081 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=30000
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118       --asf-nightlies-general-check-base=*)
 119         delete_parameter "${i}"
 120         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
 121       ;;
 122     esac
 123   done
 124 }
 125
 126 ## @description  Queue up modules for this personality
 127 ## @audience     private
 128 ## @stability    evolving
 129 ## @param        repostatus
 130 ## @param        testtype
 131 function personality_modules
 132 {
 133   local repostatus=$1
 134   local testtype=$2
 135   local extra=""
 136   local branch1jdk8=()
 137   local jdk8module=""
 138   local MODULES=("${CHANGED_MODULES[@]}")
 139
 140   yetus_info "Personality: ${repostatus} ${testtype}"
 141
 142   clear_personality_queue
 143
 144   # At a few points, hbase modules can run build, test, etc. in parallel
 145   # Let it happen. Means we'll use more CPU but should be for short bursts.
 146   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 147   extra="--threads=2 -DHBasePatchProcess"
 148   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 149     extra="${extra} -Dhttps.protocols=TLSv1.2"
 150   fi
 151
 152   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 153   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 154   # logic is not both activated within Maven.
 155   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 156     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 157   fi
 158
 159   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 160   # full source code instead. In this case, do full compiles, tests, etc instead of per
 161   # module.
 162   # Used in nightly runs.
 163   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 164   # modules if root is included. HBASE-18505
 165   if [[ "${BUILDMODE}" == "full" ]] || \
 166      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 167      [[ "${MODULES[*]}" =~ \. ]]; }; then
 168     MODULES=(.)
 169   fi
 170
 171   # If the checkstyle configs change, check everything.
 172   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 173     MODULES=(.)
 174   fi
 175
 176   if [[ ${testtype} == mvninstall ]]; then
 177     # shellcheck disable=SC2086
 178     personality_enqueue_module . ${extra}
 179     return
 180   fi
 181
 182   # This list should include any modules that require jdk8. Maven should be configured to only
 183   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 184   # module to build as yetus does if something changes in the module.  Rather than try to
 185   # figure out what jdk is in use so we can duplicate the module activation logic, just
 186   # build at the top level if anything changes in one of these modules and let maven sort it out.
 187   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 188   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 189     for jdk8module in "${branch1jdk8[@]}"; do
 190       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 191         MODULES=(.)
 192         break
 193       fi
 194     done
 195   fi
 196
 197   if [[ ${testtype} == spotbugs ]]; then
 198     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 199     # report new warnings for changed modules only.
 200     # For some reason, spotbugs on root is not working, but running on individual modules is
 201     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 202     for module in "${CHANGED_MODULES[@]}"; do
 203       # skip spotbugs on any module that lacks content in `src/main/java`
 204       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 205           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 206         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 207         continue
 208       else
 209         # shellcheck disable=SC2086
 210         personality_enqueue_module ${module} ${extra}
 211       fi
 212     done
 213     return
 214   fi
 215
 216   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 217       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 218     extra="${extra} -PerrorProne"
 219   fi
 220
 221   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 222   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 223   # tests respectively.
 224   if [[ ${testtype} == unit ]]; then
 225     local tests_arg=""
 226     get_include_exclude_tests_arg tests_arg
 227     extra="${extra} -PrunAllTests ${tests_arg}"
 228
 229     # Inject the jenkins build-id for our surefire invocations
 230     # Used by zombie detection stuff, even though we're not including that yet.
 231     if [ -n "${BUILD_ID}" ]; then
 232       extra="${extra} -Dbuild.id=${BUILD_ID}"
 233     fi
 234
 235     # If the set of changed files includes CommonFSUtils then add the hbase-server
 236     # module to the set of modules (if not already included) to be tested
 237     for f in "${CHANGED_FILES[@]}"
 238     do
 239       if [[ "${f}" =~ CommonFSUtils ]]; then
 240         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 241           MODULES+=("hbase-server")
 242         fi
 243         break
 244       fi
 245     done
 246   fi
 247
 248   for module in "${MODULES[@]}"; do
 249     # shellcheck disable=SC2086
 250     personality_enqueue_module ${module} ${extra}
 251   done
 252 }
 253
 254 ## @description places where we override the built in assumptions about what tests to run
 255 ## @audience    private
 256 ## @stability   evolving
 257 ## @param       filename of changed file
 258 function personality_file_tests
 259 {
 260   local filename=$1
 261   yetus_debug "HBase specific personality_file_tests"
 262   # If the change is to the refguide, then we don't need any builtin yetus tests
 263   # the refguide test (below) will suffice for coverage.
 264   if [[ ${filename} =~ src/main/asciidoc ]] ||
 265      [[ ${filename} =~ src/main/xslt ]]; then
 266     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 267   else
 268     # If we change our asciidoc, rebuild mvnsite
 269     if [[ ${BUILDTOOL} = maven ]]; then
 270       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 271         yetus_debug "tests/mvnsite: ${filename}"
 272         add_test mvnsite
 273       fi
 274     fi
 275     # If we change checkstyle configs, run checkstyle
 276     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 277       yetus_debug "tests/checkstyle: ${filename}"
 278       add_test checkstyle
 279     fi
 280     # fallback to checking which tests based on what yetus would do by default
 281     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 282       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 283     elif declare -f builtin_personality_file_tests >/dev/null; then
 284       builtin_personality_file_tests "${filename}"
 285     fi
 286   fi
 287 }
 288
 289 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 290 #                tests and sets given variable to arguments to be passes to maven command.
 291 ## @audience     private
 292 ## @stability    evolving
 293 ## @param        name of variable to set with maven arguments
 294 function get_include_exclude_tests_arg
 295 {
 296   local  __resultvar=$1
 297   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 298   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 299   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 300       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 301         excludes=$(cat excludes)
 302         yetus_debug "excludes=${excludes}"
 303         if [[ -n "${excludes}" ]]; then
 304           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 305         fi
 306         rm excludes
 307       else
 308         yetus_error "Wget error $? in fetching excludes file from url" \
 309              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 310       fi
 311   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 312       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 313         includes=$(cat includes)
 314         yetus_debug "includes=${includes}"
 315         if [[ -n "${includes}" ]]; then
 316           eval "${__resultvar}='-Dtest=${includes}'"
 317         fi
 318         rm includes
 319       else
 320         yetus_error "Wget error $? in fetching includes file from url" \
 321              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 322       fi
 323   else
 324     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 325     FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/output/excludes"
 326     if wget "${FLAKY_URL}" -O "excludes"; then
 327       excludes=$(cat excludes)
 328         yetus_debug "excludes=${excludes}"
 329         if [[ -n "${excludes}" ]]; then
 330           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 331         fi
 332         rm excludes
 333       else
 334         yetus_error "Wget error $? in fetching excludes file from url" \
 335              "${FLAKY_URL}. Ignoring and proceeding."
 336       fi
 337   fi
 338 }
 339
 340 ###################################################
 341 # Below here are our one-off tests specific to hbase.
 342 # TODO break them into individual files so it's easier to maintain them?
 343
 344 # TODO line length check? could ignore all java files since checkstyle gets them.
 345
 346 ###################################################
 347
 348 add_test_type refguide
 349
 350 function refguide_initialize
 351 {
 352   maven_add_install refguide
 353 }
 354
 355 function refguide_filefilter
 356 {
 357   local filename=$1
 358
 359   if [[ ${filename} =~ src/main/asciidoc ]] ||
 360      [[ ${filename} =~ src/main/xslt ]] ||
 361      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default\.xml ]]; then
 362     add_test refguide
 363   fi
 364 }
 365
 366 function refguide_rebuild
 367 {
 368   local repostatus=$1
 369   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 370   declare -i count
 371   declare pdf_output
 372
 373   if ! verify_needed_test refguide; then
 374     return 0
 375   fi
 376
 377   big_console_header "Checking we can create the ref guide on ${repostatus}"
 378
 379   start_clock
 380
 381   # disabled because "maven_executor" needs to return both command and args
 382   # shellcheck disable=2046
 383   echo_and_redirect "${logfile}" \
 384     $(maven_executor) clean site --batch-mode \
 385       -pl . \
 386       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 387       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 388
 389   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 390   if [[ ${count} -gt 0 ]]; then
 391     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 392     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 393     return 1
 394   fi
 395
 396   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 397     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 398     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 399     return 1
 400   fi
 401
 402   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 403     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 404     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 405     return 1
 406   fi
 407
 408   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 409     pdf_output="book.pdf"
 410   else
 411     pdf_output="apache_hbase_reference_guide.pdf"
 412   fi
 413
 414   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 415     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 416     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 417     return 1
 418   fi
 419
 420   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 421   if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
 422     add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
 423   else
 424     add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 425   fi
 426   return 0
 427 }
 428
 429 add_test_type shadedjars
 430
 431
 432 function shadedjars_initialize
 433 {
 434   yetus_debug "initializing shaded client checks."
 435   maven_add_install shadedjars
 436 }
 437
 438 ## @description  only run the test if java changes.
 439 ## @audience     private
 440 ## @stability    evolving
 441 ## @param        filename
 442 function shadedjars_filefilter
 443 {
 444   local filename=$1
 445
 446   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 447     add_test shadedjars
 448   fi
 449 }
 450
 451 ## @description test the shaded client artifacts
 452 ## @audience private
 453 ## @stability evolving
 454 ## @param repostatus
 455 function shadedjars_rebuild
 456 {
 457   local repostatus=$1
 458   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 459
 460   if ! verify_needed_test shadedjars; then
 461     return 0
 462   fi
 463
 464   big_console_header "Checking shaded client builds on ${repostatus}"
 465
 466   start_clock
 467
 468   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 469     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 470     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 471     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 472   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 473   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 474   # logic is not both activated within Maven.
 475   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 476     maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
 477   fi
 478
 479   # disabled because "maven_executor" needs to return both command and args
 480   # shellcheck disable=2046
 481   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 482
 483   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 484   if [[ ${count} -gt 0 ]]; then
 485     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 486     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 487     return 1
 488   fi
 489
 490   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 491   return 0
 492 }
 493
 494 ###################################################
 495
 496 add_test_type hadoopcheck
 497
 498 ## @description  hadoopcheck file filter
 499 ## @audience     private
 500 ## @stability    evolving
 501 ## @param        filename
 502 function hadoopcheck_filefilter
 503 {
 504   local filename=$1
 505
 506   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom\.xml$ ]]; then
 507     add_test hadoopcheck
 508   fi
 509 }
 510
 511 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 512 ## @audience     private
 513 ## @stability    evolving
 514 function hadoopcheck_parse_args
 515 {
 516   declare i
 517
 518   for i in "$@"; do
 519     case ${i} in
 520       --quick-hadoopcheck)
 521         delete_parameter "${i}"
 522         QUICK_HADOOPCHECK=true
 523       ;;
 524     esac
 525   done
 526 }
 527
 528 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 529 ## @audience     private
 530 ## @stability    evolving
 531 function hadoopcheck_docker_support
 532 {
 533   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 534 }
 535
 536 ## @description  hadoopcheck test
 537 ## @audience     private
 538 ## @stability    evolving
 539 ## @param        repostatus
 540 function hadoopcheck_rebuild
 541 {
 542   local repostatus=$1
 543   local hadoopver
 544   local logfile
 545   local count
 546   local result=0
 547   local hbase_hadoop2_versions
 548   local hbase_hadoop3_versions
 549
 550   if [[ "${repostatus}" = branch ]]; then
 551     return 0
 552   fi
 553
 554   if ! verify_needed_test hadoopcheck; then
 555     return 0
 556   fi
 557
 558   big_console_header "Compiling against various Hadoop versions"
 559
 560   start_clock
 561
 562   # All supported Hadoop versions that we want to test the compilation with
 563   # See the Hadoop section on prereqs in the HBase Reference Guide
 564   if [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 565     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 566     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 567       hbase_hadoop2_versions="2.7.7"
 568     else
 569       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 570     fi
 571   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 572     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 573     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 574       hbase_hadoop2_versions="2.10.0"
 575     else
 576       hbase_hadoop2_versions="2.10.0"
 577     fi
 578   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 579     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 580     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 581       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 582     else
 583       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 584     fi
 585   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 586     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 587     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 588       hbase_hadoop2_versions="2.7.7 2.8.5"
 589     else
 590       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 591     fi
 592   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 593     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 594     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 595       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 596     else
 597       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 598     fi
 599   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 600     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 601     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 602       hbase_hadoop2_versions="2.10.1"
 603     else
 604       hbase_hadoop2_versions="2.10.0 2.10.1"
 605     fi
 606   else
 607     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 608     hbase_hadoop2_versions=""
 609   fi
 610   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 611     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 612     hbase_hadoop3_versions=""
 613   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 614     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 615     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 616       hbase_hadoop3_versions="3.0.3 3.1.2"
 617     else
 618       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 619     fi
 620   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]] || [[ "${PATCH_BRANCH}" = branch-2.3 ]]; then
 621     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2/branch-2.3 rules"
 622     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 623       hbase_hadoop3_versions="3.1.2 3.2.2"
 624     else
 625       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2"
 626     fi
 627   else
 628     yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules"
 629     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 630       hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1"
 631     else
 632       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1"
 633     fi
 634   fi
 635
 636   export MAVEN_OPTS="${MAVEN_OPTS}"
 637   for hadoopver in ${hbase_hadoop2_versions}; do
 638     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 639     # disabled because "maven_executor" needs to return both command and args
 640     # shellcheck disable=2046
 641     echo_and_redirect "${logfile}" \
 642       $(maven_executor) clean install \
 643         -DskipTests -DHBasePatchProcess \
 644         -Dhadoop-two.version="${hadoopver}"
 645     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 646     if [[ ${count} -gt 0 ]]; then
 647       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 648       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 649       ((result=result+1))
 650     fi
 651   done
 652
 653   hadoop_profile=""
 654   if [[ "${PATCH_BRANCH}" = branch-2* ]]; then
 655     hadoop_profile="-Dhadoop.profile=3.0"
 656   fi
 657   for hadoopver in ${hbase_hadoop3_versions}; do
 658     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 659     # disabled because "maven_executor" needs to return both command and args
 660     # shellcheck disable=2046
 661     echo_and_redirect "${logfile}" \
 662       $(maven_executor) clean install \
 663         -DskipTests -DHBasePatchProcess \
 664         -Dhadoop-three.version="${hadoopver}" \
 665         ${hadoop_profile}
 666     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 667     if [[ ${count} -gt 0 ]]; then
 668       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 669       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 670       ((result=result+1))
 671     fi
 672   done
 673
 674   if [[ ${result} -gt 0 ]]; then
 675     return 1
 676   fi
 677
 678   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 679     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 680       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 681     else
 682       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 683     fi
 684   else
 685     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 686   fi
 687
 688   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 689   echo_and_redirect "${logfile}" \
 690     $(maven_executor) clean install \
 691       -DskipTests -DHBasePatchProcess
 692
 693   return 0
 694 }
 695
 696 ######################################
 697
 698 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 699 add_test_type hbaseprotoc
 700
 701 function hbaseprotoc_initialize
 702 {
 703   # So long as there are inter-module dependencies on the protoc modules, we
 704   # need to run a full `mvn install` before a patch can be tested.
 705   yetus_debug "initializing HBase Protoc plugin."
 706   maven_add_install hbaseprotoc
 707 }
 708
 709 ## @description  hbaseprotoc file filter
 710 ## @audience     private
 711 ## @stability    evolving
 712 ## @param        filename
 713 function hbaseprotoc_filefilter
 714 {
 715   local filename=$1
 716
 717   if [[ ${filename} =~ \.proto$ ]]; then
 718     add_test hbaseprotoc
 719   fi
 720 }
 721
 722 ## @description  check hbase proto compilation
 723 ## @audience     private
 724 ## @stability    evolving
 725 ## @param        repostatus
 726 function hbaseprotoc_rebuild
 727 {
 728   declare repostatus=$1
 729   declare i=0
 730   declare fn
 731   declare module
 732   declare logfile
 733   declare count
 734   declare result
 735
 736   if [[ "${repostatus}" = branch ]]; then
 737     return 0
 738   fi
 739
 740   if ! verify_needed_test hbaseprotoc; then
 741     return 0
 742   fi
 743
 744   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 745
 746   start_clock
 747
 748   personality_modules patch hbaseprotoc
 749   # Need to run 'install' instead of 'compile' because shading plugin
 750   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 751   # half of its process done.
 752   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 753
 754   # shellcheck disable=SC2153
 755   until [[ $i -eq "${#MODULE[@]}" ]]; do
 756     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 757       ((result=result+1))
 758       ((i=i+1))
 759       continue
 760     fi
 761     module=${MODULE[$i]}
 762     fn=$(module_file_fragment "${module}")
 763     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 764
 765     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 766
 767     if [[ ${count} -gt 0 ]]; then
 768       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 769         "${count} new protoc errors in ${module}."
 770       ((result=result+1))
 771     fi
 772     ((i=i+1))
 773   done
 774
 775   modules_messages patch hbaseprotoc true
 776   if [[ ${result} -gt 0 ]]; then
 777     return 1
 778   fi
 779   return 0
 780 }
 781
 782 ######################################
 783
 784 add_test_type hbaseanti
 785
 786 ## @description  hbaseanti file filter
 787 ## @audience     private
 788 ## @stability    evolving
 789 ## @param        filename
 790 function hbaseanti_filefilter
 791 {
 792   local filename=$1
 793
 794   if [[ ${filename} =~ \.java$ ]]; then
 795     add_test hbaseanti
 796   fi
 797 }
 798
 799 ## @description  hbaseanti patch file check
 800 ## @audience     private
 801 ## @stability    evolving
 802 ## @param        filename
 803 function hbaseanti_patchfile
 804 {
 805   local patchfile=$1
 806   local warnings
 807   local result
 808
 809   if [[ "${BUILDMODE}" = full ]]; then
 810     return 0
 811   fi
 812
 813   if ! verify_needed_test hbaseanti; then
 814     return 0
 815   fi
 816
 817   big_console_header "Checking for known anti-patterns"
 818
 819   start_clock
 820
 821   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 822   if [[ ${warnings} -gt 0 ]]; then
 823     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 824     ((result=result+1))
 825   fi
 826
 827   if [[ ${result} -gt 0 ]]; then
 828     return 1
 829   fi
 830
 831   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 832   return 0
 833 }
 834
 835 ## @description  process the javac output for generating WARNING/ERROR
 836 ## @audience     private
 837 ## @stability    evolving
 838 ## @param        input filename
 839 ## @param        output filename
 840 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 841 # This is because that the output order of the error prone warnings is not stable, so the diff
 842 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 843 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 844 # really hard to sort by file name first and then line number and column number in shell...
 845 function hbase_javac_logfilter
 846 {
 847   declare input=$1
 848   declare output=$2
 849
 850   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 851 }
 852
 853 ## This is named so that yetus will check us right after running tests.
 854 ## Essentially, we check for normal failures and then we look for zombies.
 855 #function hbase_unit_logfilter
 856 #{
 857 #  declare testtype="unit"
 858 #  declare input=$1
 859 #  declare output=$2
 860 #  declare processes
 861 #  declare process_output
 862 #  declare zombies
 863 #  declare zombie_count=0
 864 #  declare zombie_process
 865 #
 866 #  yetus_debug "in hbase-specific unit logfilter."
 867 #
 868 #  # pass-through to whatever is counting actual failures
 869 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 870 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 871 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 872 #    "${testtype}_logfilter" "${input}" "${output}"
 873 #  fi
 874 #
 875 #  start_clock
 876 #  if [ -n "${BUILD_ID}" ]; then
 877 #    yetus_debug "Checking for zombie test processes."
 878 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 879 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 880 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 881 #      yetus_debug "${processes}"
 882 #      sleep 30
 883 #      #shellcheck disable=SC2016
 884 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 885 #        # Test our zombie still running (and that it still an hbase build item)
 886 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 887 #        if [[ -n "${process_output}" ]]; then
 888 #          yetus_error "Zombie: ${process_output}"
 889 #          ((zombie_count = zombie_count + 1))
 890 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 891 #          zombies="${zombies} ${zombie_process}"
 892 #        fi
 893 #      done
 894 #    fi
 895 #    if [ "${zombie_count}" -ne 0 ]; then
 896 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 897 #      populate_test_table "zombie unit tests" "${zombies}"
 898 #    else
 899 #      yetus_info "Zombie check complete. All test runs exited normally."
 900 #      stop_clock
 901 #    fi
 902 #  else
 903 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 904 #  fi
 905 #
 906 #}