dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-25081 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=30000
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118       --asf-nightlies-general-check-base=*)
 119         delete_parameter "${i}"
 120         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
 121       ;;
 122       --build-thread=*
 123         delete_parameter "${i}"
 124         BUILD_THREAD=${i#*=}
 125       ;;
 126       --surefire-first-part-fork-count=*
 127         delete_parameter "${i}"
 128         SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
 129       ;;
 130       --surefire-second-part-fork-count=*
 131         delete_parameter "${i}"
 132         SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
 133       ;;
 134     esac
 135   done
 136 }
 137
 138 ## @description  Queue up modules for this personality
 139 ## @audience     private
 140 ## @stability    evolving
 141 ## @param        repostatus
 142 ## @param        testtype
 143 function personality_modules
 144 {
 145   local repostatus=$1
 146   local testtype=$2
 147   local extra=""
 148   local branch1jdk8=()
 149   local jdk8module=""
 150   local MODULES=("${CHANGED_MODULES[@]}")
 151
 152   yetus_info "Personality: ${repostatus} ${testtype}"
 153
 154   clear_personality_queue
 155
 156   # At a few points, hbase modules can run build, test, etc. in parallel
 157   # Let it happen. Means we'll use more CPU but should be for short bursts.
 158   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 159   if [[ -n "${BUILD_THREAD}" ]]; then
 160     extra="--threads=${BUILD_THREAD}"
 161   else
 162     extra="--threads=2"
 163   fi
 164
 165   # Set java.io.tmpdir to avoid exhausting the /tmp space
 166   # Just simply set to 'target', it is not very critical so we do not care
 167   # whether it is placed in the root directory or a sub module's directory
 168   extra="${extra} -Djava.io.tmpdir=target -DHBasePatchProcess"
 169
 170   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 171     extra="${extra} -Dhttps.protocols=TLSv1.2"
 172   fi
 173
 174   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 175   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 176   # logic is not both activated within Maven.
 177   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 178     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 179   fi
 180
 181   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 182   # full source code instead. In this case, do full compiles, tests, etc instead of per
 183   # module.
 184   # Used in nightly runs.
 185   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 186   # modules if root is included. HBASE-18505
 187   if [[ "${BUILDMODE}" == "full" ]] || \
 188      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 189      [[ "${MODULES[*]}" =~ \. ]]; }; then
 190     MODULES=(.)
 191   fi
 192
 193   # If the checkstyle configs change, check everything.
 194   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 195     MODULES=(.)
 196   fi
 197
 198   if [[ ${testtype} == mvninstall ]]; then
 199     # shellcheck disable=SC2086
 200     personality_enqueue_module . ${extra}
 201     return
 202   fi
 203
 204   # This list should include any modules that require jdk8. Maven should be configured to only
 205   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 206   # module to build as yetus does if something changes in the module.  Rather than try to
 207   # figure out what jdk is in use so we can duplicate the module activation logic, just
 208   # build at the top level if anything changes in one of these modules and let maven sort it out.
 209   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 210   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 211     for jdk8module in "${branch1jdk8[@]}"; do
 212       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 213         MODULES=(.)
 214         break
 215       fi
 216     done
 217   fi
 218
 219   if [[ ${testtype} == spotbugs ]]; then
 220     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 221     # report new warnings for changed modules only.
 222     # For some reason, spotbugs on root is not working, but running on individual modules is
 223     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 224     for module in "${CHANGED_MODULES[@]}"; do
 225       # skip spotbugs on any module that lacks content in `src/main/java`
 226       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 227           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 228         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 229         continue
 230       else
 231         # shellcheck disable=SC2086
 232         personality_enqueue_module ${module} ${extra}
 233       fi
 234     done
 235     return
 236   fi
 237
 238   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 239       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 240     extra="${extra} -PerrorProne"
 241   fi
 242
 243   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 244   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 245   # tests respectively.
 246   if [[ ${testtype} == unit ]]; then
 247     local tests_arg=""
 248     get_include_exclude_tests_arg tests_arg
 249     extra="${extra} -PrunAllTests ${tests_arg}"
 250
 251     # Inject the jenkins build-id for our surefire invocations
 252     # Used by zombie detection stuff, even though we're not including that yet.
 253     if [ -n "${BUILD_ID}" ]; then
 254       extra="${extra} -Dbuild.id=${BUILD_ID}"
 255     fi
 256
 257     # set forkCount
 258     if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
 259       extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
 260     fi
 261
 262     if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
 263       extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
 264     fi
 265
 266     # If the set of changed files includes CommonFSUtils then add the hbase-server
 267     # module to the set of modules (if not already included) to be tested
 268     for f in "${CHANGED_FILES[@]}"
 269     do
 270       if [[ "${f}" =~ CommonFSUtils ]]; then
 271         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 272           MODULES+=("hbase-server")
 273         fi
 274         break
 275       fi
 276     done
 277   fi
 278
 279   for module in "${MODULES[@]}"; do
 280     # shellcheck disable=SC2086
 281     personality_enqueue_module ${module} ${extra}
 282   done
 283 }
 284
 285 ## @description places where we override the built in assumptions about what tests to run
 286 ## @audience    private
 287 ## @stability   evolving
 288 ## @param       filename of changed file
 289 function personality_file_tests
 290 {
 291   local filename=$1
 292   yetus_debug "HBase specific personality_file_tests"
 293   # If the change is to the refguide, then we don't need any builtin yetus tests
 294   # the refguide test (below) will suffice for coverage.
 295   if [[ ${filename} =~ src/main/asciidoc ]] ||
 296      [[ ${filename} =~ src/main/xslt ]]; then
 297     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 298   else
 299     # If we change our asciidoc, rebuild mvnsite
 300     if [[ ${BUILDTOOL} = maven ]]; then
 301       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 302         yetus_debug "tests/mvnsite: ${filename}"
 303         add_test mvnsite
 304       fi
 305     fi
 306     # If we change checkstyle configs, run checkstyle
 307     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 308       yetus_debug "tests/checkstyle: ${filename}"
 309       add_test checkstyle
 310     fi
 311     # fallback to checking which tests based on what yetus would do by default
 312     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 313       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 314     elif declare -f builtin_personality_file_tests >/dev/null; then
 315       builtin_personality_file_tests "${filename}"
 316     fi
 317   fi
 318 }
 319
 320 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 321 #                tests and sets given variable to arguments to be passes to maven command.
 322 ## @audience     private
 323 ## @stability    evolving
 324 ## @param        name of variable to set with maven arguments
 325 function get_include_exclude_tests_arg
 326 {
 327   local  __resultvar=$1
 328   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 329   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 330   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 331       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 332         excludes=$(cat excludes)
 333         yetus_debug "excludes=${excludes}"
 334         if [[ -n "${excludes}" ]]; then
 335           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 336         fi
 337         rm excludes
 338       else
 339         yetus_error "Wget error $? in fetching excludes file from url" \
 340              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 341       fi
 342   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 343       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 344         includes=$(cat includes)
 345         yetus_debug "includes=${includes}"
 346         if [[ -n "${includes}" ]]; then
 347           eval "${__resultvar}='-Dtest=${includes}'"
 348         fi
 349         rm includes
 350       else
 351         yetus_error "Wget error $? in fetching includes file from url" \
 352              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 353       fi
 354   else
 355     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 356     FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/output/excludes"
 357     if wget "${FLAKY_URL}" -O "excludes"; then
 358       excludes=$(cat excludes)
 359         yetus_debug "excludes=${excludes}"
 360         if [[ -n "${excludes}" ]]; then
 361           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 362         fi
 363         rm excludes
 364       else
 365         yetus_error "Wget error $? in fetching excludes file from url" \
 366              "${FLAKY_URL}. Ignoring and proceeding."
 367       fi
 368   fi
 369 }
 370
 371 ###################################################
 372 # Below here are our one-off tests specific to hbase.
 373 # TODO break them into individual files so it's easier to maintain them?
 374
 375 # TODO line length check? could ignore all java files since checkstyle gets them.
 376
 377 ###################################################
 378
 379 add_test_type refguide
 380
 381 function refguide_initialize
 382 {
 383   maven_add_install refguide
 384 }
 385
 386 function refguide_filefilter
 387 {
 388   local filename=$1
 389
 390   if [[ ${filename} =~ src/main/asciidoc ]] ||
 391      [[ ${filename} =~ src/main/xslt ]] ||
 392      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default\.xml ]]; then
 393     add_test refguide
 394   fi
 395 }
 396
 397 function refguide_rebuild
 398 {
 399   local repostatus=$1
 400   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 401   declare -i count
 402   declare pdf_output
 403
 404   if ! verify_needed_test refguide; then
 405     return 0
 406   fi
 407
 408   big_console_header "Checking we can create the ref guide on ${repostatus}"
 409
 410   start_clock
 411
 412   # disabled because "maven_executor" needs to return both command and args
 413   # shellcheck disable=2046
 414   echo_and_redirect "${logfile}" \
 415     $(maven_executor) clean site --batch-mode \
 416       -pl . \
 417       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 418       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 419
 420   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 421   if [[ ${count} -gt 0 ]]; then
 422     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 423     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 424     return 1
 425   fi
 426
 427   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 428     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 429     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 430     return 1
 431   fi
 432
 433   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 434     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 435     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 436     return 1
 437   fi
 438
 439   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 440     pdf_output="book.pdf"
 441   else
 442     pdf_output="apache_hbase_reference_guide.pdf"
 443   fi
 444
 445   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 446     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 447     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 448     return 1
 449   fi
 450
 451   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 452   if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
 453     add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
 454   else
 455     add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 456   fi
 457   return 0
 458 }
 459
 460 add_test_type shadedjars
 461
 462
 463 function shadedjars_initialize
 464 {
 465   yetus_debug "initializing shaded client checks."
 466   maven_add_install shadedjars
 467 }
 468
 469 ## @description  only run the test if java changes.
 470 ## @audience     private
 471 ## @stability    evolving
 472 ## @param        filename
 473 function shadedjars_filefilter
 474 {
 475   local filename=$1
 476
 477   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 478     add_test shadedjars
 479   fi
 480 }
 481
 482 ## @description test the shaded client artifacts
 483 ## @audience private
 484 ## @stability evolving
 485 ## @param repostatus
 486 function shadedjars_rebuild
 487 {
 488   local repostatus=$1
 489   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 490
 491   if ! verify_needed_test shadedjars; then
 492     return 0
 493   fi
 494
 495   big_console_header "Checking shaded client builds on ${repostatus}"
 496
 497   start_clock
 498
 499   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 500     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 501     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 502     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 503   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 504   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 505   # logic is not both activated within Maven.
 506   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 507     maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
 508   fi
 509
 510   # disabled because "maven_executor" needs to return both command and args
 511   # shellcheck disable=2046
 512   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 513
 514   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 515   if [[ ${count} -gt 0 ]]; then
 516     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 517     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 518     return 1
 519   fi
 520
 521   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 522   return 0
 523 }
 524
 525 ###################################################
 526
 527 add_test_type hadoopcheck
 528
 529 ## @description  hadoopcheck file filter
 530 ## @audience     private
 531 ## @stability    evolving
 532 ## @param        filename
 533 function hadoopcheck_filefilter
 534 {
 535   local filename=$1
 536
 537   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom\.xml$ ]]; then
 538     add_test hadoopcheck
 539   fi
 540 }
 541
 542 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 543 ## @audience     private
 544 ## @stability    evolving
 545 function hadoopcheck_parse_args
 546 {
 547   declare i
 548
 549   for i in "$@"; do
 550     case ${i} in
 551       --quick-hadoopcheck)
 552         delete_parameter "${i}"
 553         QUICK_HADOOPCHECK=true
 554       ;;
 555     esac
 556   done
 557 }
 558
 559 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 560 ## @audience     private
 561 ## @stability    evolving
 562 function hadoopcheck_docker_support
 563 {
 564   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 565 }
 566
 567 ## @description  hadoopcheck test
 568 ## @audience     private
 569 ## @stability    evolving
 570 ## @param        repostatus
 571 function hadoopcheck_rebuild
 572 {
 573   local repostatus=$1
 574   local hadoopver
 575   local logfile
 576   local count
 577   local result=0
 578   local hbase_hadoop2_versions
 579   local hbase_hadoop3_versions
 580
 581   if [[ "${repostatus}" = branch ]]; then
 582     return 0
 583   fi
 584
 585   if ! verify_needed_test hadoopcheck; then
 586     return 0
 587   fi
 588
 589   big_console_header "Compiling against various Hadoop versions"
 590
 591   start_clock
 592
 593   # All supported Hadoop versions that we want to test the compilation with
 594   # See the Hadoop section on prereqs in the HBase Reference Guide
 595   if [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 596     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 597     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 598       hbase_hadoop2_versions="2.7.7"
 599     else
 600       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 601     fi
 602   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 603     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 604     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 605       hbase_hadoop2_versions="2.10.0"
 606     else
 607       hbase_hadoop2_versions="2.10.0"
 608     fi
 609   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 610     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 611     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 612       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 613     else
 614       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 615     fi
 616   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 617     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 618     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 619       hbase_hadoop2_versions="2.7.7 2.8.5"
 620     else
 621       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 622     fi
 623   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 624     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 625     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 626       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 627     else
 628       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 629     fi
 630   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 631     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 632     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 633       hbase_hadoop2_versions="2.10.1"
 634     else
 635       hbase_hadoop2_versions="2.10.0 2.10.1"
 636     fi
 637   else
 638     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 639     hbase_hadoop2_versions=""
 640   fi
 641   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 642     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 643     hbase_hadoop3_versions=""
 644   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 645     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 646     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 647       hbase_hadoop3_versions="3.0.3 3.1.2"
 648     else
 649       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 650     fi
 651   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]] || [[ "${PATCH_BRANCH}" = branch-2.3 ]]; then
 652     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2/branch-2.3 rules"
 653     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 654       hbase_hadoop3_versions="3.1.2 3.2.2"
 655     else
 656       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2"
 657     fi
 658   else
 659     yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules"
 660     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 661       hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1"
 662     else
 663       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1"
 664     fi
 665   fi
 666
 667   export MAVEN_OPTS="${MAVEN_OPTS}"
 668   for hadoopver in ${hbase_hadoop2_versions}; do
 669     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 670     # disabled because "maven_executor" needs to return both command and args
 671     # shellcheck disable=2046
 672     echo_and_redirect "${logfile}" \
 673       $(maven_executor) clean install \
 674         -DskipTests -DHBasePatchProcess \
 675         -Dhadoop-two.version="${hadoopver}"
 676     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 677     if [[ ${count} -gt 0 ]]; then
 678       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 679       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 680       ((result=result+1))
 681     fi
 682   done
 683
 684   hadoop_profile=""
 685   if [[ "${PATCH_BRANCH}" = branch-2* ]]; then
 686     hadoop_profile="-Dhadoop.profile=3.0"
 687   fi
 688   for hadoopver in ${hbase_hadoop3_versions}; do
 689     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 690     # disabled because "maven_executor" needs to return both command and args
 691     # shellcheck disable=2046
 692     echo_and_redirect "${logfile}" \
 693       $(maven_executor) clean install \
 694         -DskipTests -DHBasePatchProcess \
 695         -Dhadoop-three.version="${hadoopver}" \
 696         ${hadoop_profile}
 697     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 698     if [[ ${count} -gt 0 ]]; then
 699       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 700       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 701       ((result=result+1))
 702     fi
 703   done
 704
 705   if [[ ${result} -gt 0 ]]; then
 706     return 1
 707   fi
 708
 709   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 710     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 711       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 712     else
 713       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 714     fi
 715   else
 716     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 717   fi
 718
 719   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 720   echo_and_redirect "${logfile}" \
 721     $(maven_executor) clean install \
 722       -DskipTests -DHBasePatchProcess
 723
 724   return 0
 725 }
 726
 727 ######################################
 728
 729 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 730 add_test_type hbaseprotoc
 731
 732 function hbaseprotoc_initialize
 733 {
 734   # So long as there are inter-module dependencies on the protoc modules, we
 735   # need to run a full `mvn install` before a patch can be tested.
 736   yetus_debug "initializing HBase Protoc plugin."
 737   maven_add_install hbaseprotoc
 738 }
 739
 740 ## @description  hbaseprotoc file filter
 741 ## @audience     private
 742 ## @stability    evolving
 743 ## @param        filename
 744 function hbaseprotoc_filefilter
 745 {
 746   local filename=$1
 747
 748   if [[ ${filename} =~ \.proto$ ]]; then
 749     add_test hbaseprotoc
 750   fi
 751 }
 752
 753 ## @description  check hbase proto compilation
 754 ## @audience     private
 755 ## @stability    evolving
 756 ## @param        repostatus
 757 function hbaseprotoc_rebuild
 758 {
 759   declare repostatus=$1
 760   declare i=0
 761   declare fn
 762   declare module
 763   declare logfile
 764   declare count
 765   declare result
 766
 767   if [[ "${repostatus}" = branch ]]; then
 768     return 0
 769   fi
 770
 771   if ! verify_needed_test hbaseprotoc; then
 772     return 0
 773   fi
 774
 775   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 776
 777   start_clock
 778
 779   personality_modules patch hbaseprotoc
 780   # Need to run 'install' instead of 'compile' because shading plugin
 781   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 782   # half of its process done.
 783   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 784
 785   # shellcheck disable=SC2153
 786   until [[ $i -eq "${#MODULE[@]}" ]]; do
 787     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 788       ((result=result+1))
 789       ((i=i+1))
 790       continue
 791     fi
 792     module=${MODULE[$i]}
 793     fn=$(module_file_fragment "${module}")
 794     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 795
 796     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 797
 798     if [[ ${count} -gt 0 ]]; then
 799       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 800         "${count} new protoc errors in ${module}."
 801       ((result=result+1))
 802     fi
 803     ((i=i+1))
 804   done
 805
 806   modules_messages patch hbaseprotoc true
 807   if [[ ${result} -gt 0 ]]; then
 808     return 1
 809   fi
 810   return 0
 811 }
 812
 813 ######################################
 814
 815 add_test_type hbaseanti
 816
 817 ## @description  hbaseanti file filter
 818 ## @audience     private
 819 ## @stability    evolving
 820 ## @param        filename
 821 function hbaseanti_filefilter
 822 {
 823   local filename=$1
 824
 825   if [[ ${filename} =~ \.java$ ]]; then
 826     add_test hbaseanti
 827   fi
 828 }
 829
 830 ## @description  hbaseanti patch file check
 831 ## @audience     private
 832 ## @stability    evolving
 833 ## @param        filename
 834 function hbaseanti_patchfile
 835 {
 836   local patchfile=$1
 837   local warnings
 838   local result
 839
 840   if [[ "${BUILDMODE}" = full ]]; then
 841     return 0
 842   fi
 843
 844   if ! verify_needed_test hbaseanti; then
 845     return 0
 846   fi
 847
 848   big_console_header "Checking for known anti-patterns"
 849
 850   start_clock
 851
 852   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 853   if [[ ${warnings} -gt 0 ]]; then
 854     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 855     ((result=result+1))
 856   fi
 857
 858   if [[ ${result} -gt 0 ]]; then
 859     return 1
 860   fi
 861
 862   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 863   return 0
 864 }
 865
 866 ## @description  process the javac output for generating WARNING/ERROR
 867 ## @audience     private
 868 ## @stability    evolving
 869 ## @param        input filename
 870 ## @param        output filename
 871 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 872 # This is because that the output order of the error prone warnings is not stable, so the diff
 873 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 874 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 875 # really hard to sort by file name first and then line number and column number in shell...
 876 function hbase_javac_logfilter
 877 {
 878   declare input=$1
 879   declare output=$2
 880
 881   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 882 }
 883
 884 ## This is named so that yetus will check us right after running tests.
 885 ## Essentially, we check for normal failures and then we look for zombies.
 886 #function hbase_unit_logfilter
 887 #{
 888 #  declare testtype="unit"
 889 #  declare input=$1
 890 #  declare output=$2
 891 #  declare processes
 892 #  declare process_output
 893 #  declare zombies
 894 #  declare zombie_count=0
 895 #  declare zombie_process
 896 #
 897 #  yetus_debug "in hbase-specific unit logfilter."
 898 #
 899 #  # pass-through to whatever is counting actual failures
 900 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 901 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 902 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 903 #    "${testtype}_logfilter" "${input}" "${output}"
 904 #  fi
 905 #
 906 #  start_clock
 907 #  if [ -n "${BUILD_ID}" ]; then
 908 #    yetus_debug "Checking for zombie test processes."
 909 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 910 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 911 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 912 #      yetus_debug "${processes}"
 913 #      sleep 30
 914 #      #shellcheck disable=SC2016
 915 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 916 #        # Test our zombie still running (and that it still an hbase build item)
 917 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 918 #        if [[ -n "${process_output}" ]]; then
 919 #          yetus_error "Zombie: ${process_output}"
 920 #          ((zombie_count = zombie_count + 1))
 921 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 922 #          zombies="${zombies} ${zombie_process}"
 923 #        fi
 924 #      done
 925 #    fi
 926 #    if [ "${zombie_count}" -ne 0 ]; then
 927 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 928 #      populate_test_table "zombie unit tests" "${zombies}"
 929 #    else
 930 #      yetus_info "Zombie check complete. All test runs exited normally."
 931 #      stop_clock
 932 #    fi
 933 #  else
 934 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 935 #  fi
 936 #
 937 #}