dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 # work around yetus overwriting JAVA_HOME from our docker image
  49 function docker_do_env_adds
  50 {
  51   declare k
  52
  53   for k in "${DOCKER_EXTRAENVS[@]}"; do
  54     if [[ "JAVA_HOME" == "${k}" ]]; then
  55       if [ -n "${JAVA_HOME}" ]; then
  56         DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
  57       fi
  58     else
  59       DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
  60     fi
  61   done
  62 }
  63
  64
  65 ## @description  Globals specific to this personality
  66 ## @audience     private
  67 ## @stability    evolving
  68 function personality_globals
  69 {
  70   BUILDTOOL=maven
  71   #shellcheck disable=SC2034
  72   PROJECT_NAME=hbase
  73   #shellcheck disable=SC2034
  74   PATCH_BRANCH_DEFAULT=master
  75   #shellcheck disable=SC2034
  76   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  77   #shellcheck disable=SC2034
  78   GITHUB_REPO="apache/hbase"
  79
  80   # TODO use PATCH_BRANCH to select jdk versions to use.
  81
  82   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  83   # Up it. See HBASE-25081 for how we arrived at this number.
  84   #shellcheck disable=SC2034
  85   PROC_LIMIT=30000
  86
  87   # Set docker container to run with 20g. Default is 4g in yetus.
  88   # See HBASE-19902 for how we arrived at 20g.
  89   #shellcheck disable=SC2034
  90   DOCKERMEMLIMIT=20g
  91 }
  92
  93 ## @description  Parse extra arguments required by personalities, if any.
  94 ## @audience     private
  95 ## @stability    evolving
  96 function personality_parse_args
  97 {
  98   declare i
  99
 100   for i in "$@"; do
 101     case ${i} in
 102       --exclude-tests-url=*)
 103         delete_parameter "${i}"
 104         EXCLUDE_TESTS_URL=${i#*=}
 105       ;;
 106       --include-tests-url=*)
 107         delete_parameter "${i}"
 108         INCLUDE_TESTS_URL=${i#*=}
 109       ;;
 110       --hadoop-profile=*)
 111         delete_parameter "${i}"
 112         HADOOP_PROFILE=${i#*=}
 113       ;;
 114       --skip-errorprone)
 115         delete_parameter "${i}"
 116         SKIP_ERRORPRONE=true
 117       ;;
 118       --asf-nightlies-general-check-base=*)
 119         delete_parameter "${i}"
 120         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
 121       ;;
 122       --build-thread=*)
 123         delete_parameter "${i}"
 124         BUILD_THREAD=${i#*=}
 125       ;;
 126       --surefire-first-part-fork-count=*)
 127         delete_parameter "${i}"
 128         SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
 129       ;;
 130       --surefire-second-part-fork-count=*)
 131         delete_parameter "${i}"
 132         SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
 133       ;;
 134     esac
 135   done
 136 }
 137
 138 ## @description  Queue up modules for this personality
 139 ## @audience     private
 140 ## @stability    evolving
 141 ## @param        repostatus
 142 ## @param        testtype
 143 function personality_modules
 144 {
 145   local repostatus=$1
 146   local testtype=$2
 147   local extra=""
 148   local branch1jdk8=()
 149   local jdk8module=""
 150   local MODULES=("${CHANGED_MODULES[@]}")
 151
 152   yetus_info "Personality: ${repostatus} ${testtype}"
 153
 154   clear_personality_queue
 155
 156   # At a few points, hbase modules can run build, test, etc. in parallel
 157   # Let it happen. Means we'll use more CPU but should be for short bursts.
 158   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
 159   if [[ -n "${BUILD_THREAD}" ]]; then
 160     extra="--threads=${BUILD_THREAD}"
 161   else
 162     extra="--threads=2"
 163   fi
 164
 165   # Set java.io.tmpdir to avoid exhausting the /tmp space
 166   # Just simply set to 'target', it is not very critical so we do not care
 167   # whether it is placed in the root directory or a sub module's directory
 168   # let's make it absolute
 169   tmpdir=$(realpath target)
 170   extra="${extra} -Djava.io.tmpdir=${tmpdir} -DHBasePatchProcess"
 171
 172   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 173     extra="${extra} -Dhttps.protocols=TLSv1.2"
 174   fi
 175
 176   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 177   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 178   # logic is not both activated within Maven.
 179   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 180     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 181   fi
 182
 183   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 184   # full source code instead. In this case, do full compiles, tests, etc instead of per
 185   # module.
 186   # Used in nightly runs.
 187   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 188   # modules if root is included. HBASE-18505
 189   if [[ "${BUILDMODE}" == "full" ]] || \
 190      { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
 191      [[ "${MODULES[*]}" =~ \. ]]; }; then
 192     MODULES=(.)
 193   fi
 194
 195   # If the checkstyle configs change, check everything.
 196   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 197     MODULES=(.)
 198   fi
 199
 200   if [[ ${testtype} == mvninstall ]]; then
 201     # shellcheck disable=SC2086
 202     personality_enqueue_module . ${extra}
 203     return
 204   fi
 205
 206   # This list should include any modules that require jdk8. Maven should be configured to only
 207   # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
 208   # module to build as yetus does if something changes in the module.  Rather than try to
 209   # figure out what jdk is in use so we can duplicate the module activation logic, just
 210   # build at the top level if anything changes in one of these modules and let maven sort it out.
 211   branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
 212   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 213     for jdk8module in "${branch1jdk8[@]}"; do
 214       if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
 215         MODULES=(.)
 216         break
 217       fi
 218     done
 219   fi
 220
 221   if [[ ${testtype} == spotbugs ]]; then
 222     # Run spotbugs on each module individually to diff pre-patch and post-patch results and
 223     # report new warnings for changed modules only.
 224     # For some reason, spotbugs on root is not working, but running on individual modules is
 225     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 226     for module in "${CHANGED_MODULES[@]}"; do
 227       # skip spotbugs on any module that lacks content in `src/main/java`
 228       if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
 229           -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
 230         yetus_debug "no java files found under ${module}/src/main/java. skipping."
 231         continue
 232       else
 233         # shellcheck disable=SC2086
 234         personality_enqueue_module ${module} ${extra}
 235       fi
 236     done
 237     return
 238   fi
 239
 240   if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
 241       [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
 242     extra="${extra} -PerrorProne"
 243   fi
 244
 245   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 246   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 247   # tests respectively.
 248   if [[ ${testtype} == unit ]]; then
 249     local tests_arg=""
 250     get_include_exclude_tests_arg tests_arg
 251     extra="${extra} -PrunAllTests ${tests_arg}"
 252
 253     # Inject the jenkins build-id for our surefire invocations
 254     # Used by zombie detection stuff, even though we're not including that yet.
 255     if [ -n "${BUILD_ID}" ]; then
 256       extra="${extra} -Dbuild.id=${BUILD_ID}"
 257     fi
 258
 259     # set forkCount
 260     if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
 261       extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
 262     fi
 263
 264     if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
 265       extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
 266     fi
 267
 268     # If the set of changed files includes CommonFSUtils then add the hbase-server
 269     # module to the set of modules (if not already included) to be tested
 270     for f in "${CHANGED_FILES[@]}"
 271     do
 272       if [[ "${f}" =~ CommonFSUtils ]]; then
 273         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 274           MODULES+=("hbase-server")
 275         fi
 276         break
 277       fi
 278     done
 279   fi
 280
 281   for module in "${MODULES[@]}"; do
 282     # shellcheck disable=SC2086
 283     personality_enqueue_module ${module} ${extra}
 284   done
 285 }
 286
 287 ## @description places where we override the built in assumptions about what tests to run
 288 ## @audience    private
 289 ## @stability   evolving
 290 ## @param       filename of changed file
 291 function personality_file_tests
 292 {
 293   local filename=$1
 294   yetus_debug "HBase specific personality_file_tests"
 295   # If the change is to the refguide, then we don't need any builtin yetus tests
 296   # the refguide test (below) will suffice for coverage.
 297   if [[ ${filename} =~ src/main/asciidoc ]] ||
 298      [[ ${filename} =~ src/main/xslt ]]; then
 299     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 300   else
 301     # If we change our asciidoc, rebuild mvnsite
 302     if [[ ${BUILDTOOL} = maven ]]; then
 303       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 304         yetus_debug "tests/mvnsite: ${filename}"
 305         add_test mvnsite
 306       fi
 307     fi
 308     # If we change checkstyle configs, run checkstyle
 309     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 310       yetus_debug "tests/checkstyle: ${filename}"
 311       add_test checkstyle
 312     fi
 313     # fallback to checking which tests based on what yetus would do by default
 314     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 315       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 316     elif declare -f builtin_personality_file_tests >/dev/null; then
 317       builtin_personality_file_tests "${filename}"
 318     fi
 319   fi
 320 }
 321
 322 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 323 #                tests and sets given variable to arguments to be passes to maven command.
 324 ## @audience     private
 325 ## @stability    evolving
 326 ## @param        name of variable to set with maven arguments
 327 function get_include_exclude_tests_arg
 328 {
 329   local  __resultvar=$1
 330   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 331   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 332   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 333       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 334         excludes=$(cat excludes)
 335         yetus_debug "excludes=${excludes}"
 336         if [[ -n "${excludes}" ]]; then
 337           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 338         fi
 339         rm excludes
 340       else
 341         yetus_error "Wget error $? in fetching excludes file from url" \
 342              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 343       fi
 344   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 345       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 346         includes=$(cat includes)
 347         yetus_debug "includes=${includes}"
 348         if [[ -n "${includes}" ]]; then
 349           eval "${__resultvar}='-Dtest=${includes}'"
 350         fi
 351         rm includes
 352       else
 353         yetus_error "Wget error $? in fetching includes file from url" \
 354              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 355       fi
 356   else
 357     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 358     FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/output/excludes"
 359     if wget "${FLAKY_URL}" -O "excludes"; then
 360       excludes=$(cat excludes)
 361         yetus_debug "excludes=${excludes}"
 362         if [[ -n "${excludes}" ]]; then
 363           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 364         fi
 365         rm excludes
 366       else
 367         yetus_error "Wget error $? in fetching excludes file from url" \
 368              "${FLAKY_URL}. Ignoring and proceeding."
 369       fi
 370   fi
 371 }
 372
 373 ###################################################
 374 # Below here are our one-off tests specific to hbase.
 375 # TODO break them into individual files so it's easier to maintain them?
 376
 377 # TODO line length check? could ignore all java files since checkstyle gets them.
 378
 379 ###################################################
 380
 381 add_test_type refguide
 382
 383 function refguide_initialize
 384 {
 385   maven_add_install refguide
 386 }
 387
 388 function refguide_filefilter
 389 {
 390   local filename=$1
 391
 392   if [[ ${filename} =~ src/main/asciidoc ]] ||
 393      [[ ${filename} =~ src/main/xslt ]] ||
 394      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default\.xml ]]; then
 395     add_test refguide
 396   fi
 397 }
 398
 399 function refguide_rebuild
 400 {
 401   local repostatus=$1
 402   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 403   declare -i count
 404   declare pdf_output
 405
 406   if ! verify_needed_test refguide; then
 407     return 0
 408   fi
 409
 410   big_console_header "Checking we can create the ref guide on ${repostatus}"
 411
 412   start_clock
 413
 414   # disabled because "maven_executor" needs to return both command and args
 415   # shellcheck disable=2046
 416   echo_and_redirect "${logfile}" \
 417     $(maven_executor) clean site --batch-mode \
 418       -pl . \
 419       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 420       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 421
 422   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 423   if [[ ${count} -gt 0 ]]; then
 424     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 425     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 426     return 1
 427   fi
 428
 429   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 430     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 431     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 432     return 1
 433   fi
 434
 435   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 436     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 437     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 438     return 1
 439   fi
 440
 441   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 442     pdf_output="book.pdf"
 443   else
 444     pdf_output="apache_hbase_reference_guide.pdf"
 445   fi
 446
 447   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 448     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 449     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 450     return 1
 451   fi
 452
 453   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 454   if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
 455     add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
 456   else
 457     add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 458   fi
 459   return 0
 460 }
 461
 462 add_test_type shadedjars
 463
 464
 465 function shadedjars_initialize
 466 {
 467   yetus_debug "initializing shaded client checks."
 468   maven_add_install shadedjars
 469 }
 470
 471 ## @description  only run the test if java changes.
 472 ## @audience     private
 473 ## @stability    evolving
 474 ## @param        filename
 475 function shadedjars_filefilter
 476 {
 477   local filename=$1
 478
 479   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 480     add_test shadedjars
 481   fi
 482 }
 483
 484 ## @description test the shaded client artifacts
 485 ## @audience private
 486 ## @stability evolving
 487 ## @param repostatus
 488 function shadedjars_rebuild
 489 {
 490   local repostatus=$1
 491   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 492
 493   if ! verify_needed_test shadedjars; then
 494     return 0
 495   fi
 496
 497   big_console_header "Checking shaded client builds on ${repostatus}"
 498
 499   start_clock
 500
 501   local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
 502     '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
 503     '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
 504     '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
 505   # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
 506   # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
 507   # logic is not both activated within Maven.
 508   if [[ -n "${HADOOP_PROFILE}" ]] && [[ "${PATCH_BRANCH}" = branch-2* ]] ; then
 509     maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
 510   fi
 511
 512   # disabled because "maven_executor" needs to return both command and args
 513   # shellcheck disable=2046
 514   echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
 515
 516   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 517   if [[ ${count} -gt 0 ]]; then
 518     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 519     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 520     return 1
 521   fi
 522
 523   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 524   return 0
 525 }
 526
 527 ###################################################
 528
 529 add_test_type hadoopcheck
 530
 531 ## @description  hadoopcheck file filter
 532 ## @audience     private
 533 ## @stability    evolving
 534 ## @param        filename
 535 function hadoopcheck_filefilter
 536 {
 537   local filename=$1
 538
 539   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom\.xml$ ]]; then
 540     add_test hadoopcheck
 541   fi
 542 }
 543
 544 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 545 ## @audience     private
 546 ## @stability    evolving
 547 function hadoopcheck_parse_args
 548 {
 549   declare i
 550
 551   for i in "$@"; do
 552     case ${i} in
 553       --quick-hadoopcheck)
 554         delete_parameter "${i}"
 555         QUICK_HADOOPCHECK=true
 556       ;;
 557     esac
 558   done
 559 }
 560
 561 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 562 ## @audience     private
 563 ## @stability    evolving
 564 function hadoopcheck_docker_support
 565 {
 566   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 567 }
 568
 569 ## @description  hadoopcheck test
 570 ## @audience     private
 571 ## @stability    evolving
 572 ## @param        repostatus
 573 function hadoopcheck_rebuild
 574 {
 575   local repostatus=$1
 576   local hadoopver
 577   local logfile
 578   local count
 579   local result=0
 580   local hbase_hadoop2_versions
 581   local hbase_hadoop3_versions
 582
 583   if [[ "${repostatus}" = branch ]]; then
 584     return 0
 585   fi
 586
 587   if ! verify_needed_test hadoopcheck; then
 588     return 0
 589   fi
 590
 591   big_console_header "Compiling against various Hadoop versions"
 592
 593   start_clock
 594
 595   # All supported Hadoop versions that we want to test the compilation with
 596   # See the Hadoop section on prereqs in the HBase Reference Guide
 597   if [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
 598     yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
 599     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 600       hbase_hadoop2_versions="2.7.7"
 601     else
 602       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
 603     fi
 604   elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
 605     yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
 606     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 607       hbase_hadoop2_versions="2.10.0"
 608     else
 609       hbase_hadoop2_versions="2.10.0"
 610     fi
 611   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 612     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 613     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 614       hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
 615     else
 616       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 617     fi
 618   elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 619     yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
 620     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 621       hbase_hadoop2_versions="2.7.7 2.8.5"
 622     else
 623       hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
 624     fi
 625   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
 626     yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
 627     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 628       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 629     else
 630       hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
 631     fi
 632   elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
 633     yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
 634     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 635       hbase_hadoop2_versions="2.10.1"
 636     else
 637       hbase_hadoop2_versions="2.10.0 2.10.1"
 638     fi
 639   else
 640     yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
 641     hbase_hadoop2_versions=""
 642   fi
 643   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 644     yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
 645     hbase_hadoop3_versions=""
 646   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
 647     yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
 648     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 649       hbase_hadoop3_versions="3.0.3 3.1.2"
 650     else
 651       hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
 652     fi
 653   elif [[ "${PATCH_BRANCH}" = branch-2.2 ]] || [[ "${PATCH_BRANCH}" = branch-2.3 ]]; then
 654     yetus_info "Setting Hadoop 3 versions to test based on branch-2.2/branch-2.3 rules"
 655     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 656       hbase_hadoop3_versions="3.1.2 3.2.2"
 657     else
 658       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2"
 659     fi
 660   else
 661     yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules"
 662     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 663       hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1"
 664     else
 665       hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1"
 666     fi
 667   fi
 668
 669   export MAVEN_OPTS="${MAVEN_OPTS}"
 670   for hadoopver in ${hbase_hadoop2_versions}; do
 671     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 672     # disabled because "maven_executor" needs to return both command and args
 673     # shellcheck disable=2046
 674     echo_and_redirect "${logfile}" \
 675       $(maven_executor) clean install \
 676         -DskipTests -DHBasePatchProcess \
 677         -Dhadoop-two.version="${hadoopver}"
 678     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 679     if [[ ${count} -gt 0 ]]; then
 680       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 681       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 682       ((result=result+1))
 683     fi
 684   done
 685
 686   hadoop_profile=""
 687   if [[ "${PATCH_BRANCH}" = branch-2* ]]; then
 688     hadoop_profile="-Dhadoop.profile=3.0"
 689   fi
 690   for hadoopver in ${hbase_hadoop3_versions}; do
 691     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 692     # disabled because "maven_executor" needs to return both command and args
 693     # shellcheck disable=2046
 694     echo_and_redirect "${logfile}" \
 695       $(maven_executor) clean install \
 696         -DskipTests -DHBasePatchProcess \
 697         -Dhadoop-three.version="${hadoopver}" \
 698         ${hadoop_profile}
 699     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 700     if [[ ${count} -gt 0 ]]; then
 701       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 702       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 703       ((result=result+1))
 704     fi
 705   done
 706
 707   if [[ ${result} -gt 0 ]]; then
 708     return 1
 709   fi
 710
 711   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 712     if [[ -n "${hbase_hadoop2_versions}" ]]; then
 713       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 714     else
 715       add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
 716     fi
 717   else
 718     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 719   fi
 720
 721   logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
 722   echo_and_redirect "${logfile}" \
 723     $(maven_executor) clean install \
 724       -DskipTests -DHBasePatchProcess
 725
 726   return 0
 727 }
 728
 729 ######################################
 730
 731 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 732 add_test_type hbaseprotoc
 733
 734 function hbaseprotoc_initialize
 735 {
 736   # So long as there are inter-module dependencies on the protoc modules, we
 737   # need to run a full `mvn install` before a patch can be tested.
 738   yetus_debug "initializing HBase Protoc plugin."
 739   maven_add_install hbaseprotoc
 740 }
 741
 742 ## @description  hbaseprotoc file filter
 743 ## @audience     private
 744 ## @stability    evolving
 745 ## @param        filename
 746 function hbaseprotoc_filefilter
 747 {
 748   local filename=$1
 749
 750   if [[ ${filename} =~ \.proto$ ]]; then
 751     add_test hbaseprotoc
 752   fi
 753 }
 754
 755 ## @description  check hbase proto compilation
 756 ## @audience     private
 757 ## @stability    evolving
 758 ## @param        repostatus
 759 function hbaseprotoc_rebuild
 760 {
 761   declare repostatus=$1
 762   declare i=0
 763   declare fn
 764   declare module
 765   declare logfile
 766   declare count
 767   declare result
 768
 769   if [[ "${repostatus}" = branch ]]; then
 770     return 0
 771   fi
 772
 773   if ! verify_needed_test hbaseprotoc; then
 774     return 0
 775   fi
 776
 777   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 778
 779   start_clock
 780
 781   personality_modules patch hbaseprotoc
 782   # Need to run 'install' instead of 'compile' because shading plugin
 783   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 784   # half of its process done.
 785   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 786
 787   # shellcheck disable=SC2153
 788   until [[ $i -eq "${#MODULE[@]}" ]]; do
 789     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 790       ((result=result+1))
 791       ((i=i+1))
 792       continue
 793     fi
 794     module=${MODULE[$i]}
 795     fn=$(module_file_fragment "${module}")
 796     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 797
 798     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 799
 800     if [[ ${count} -gt 0 ]]; then
 801       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 802         "${count} new protoc errors in ${module}."
 803       ((result=result+1))
 804     fi
 805     ((i=i+1))
 806   done
 807
 808   modules_messages patch hbaseprotoc true
 809   if [[ ${result} -gt 0 ]]; then
 810     return 1
 811   fi
 812   return 0
 813 }
 814
 815 ######################################
 816
 817 add_test_type hbaseanti
 818
 819 ## @description  hbaseanti file filter
 820 ## @audience     private
 821 ## @stability    evolving
 822 ## @param        filename
 823 function hbaseanti_filefilter
 824 {
 825   local filename=$1
 826
 827   if [[ ${filename} =~ \.java$ ]]; then
 828     add_test hbaseanti
 829   fi
 830 }
 831
 832 ## @description  hbaseanti patch file check
 833 ## @audience     private
 834 ## @stability    evolving
 835 ## @param        filename
 836 function hbaseanti_patchfile
 837 {
 838   local patchfile=$1
 839   local warnings
 840   local result
 841
 842   if [[ "${BUILDMODE}" = full ]]; then
 843     return 0
 844   fi
 845
 846   if ! verify_needed_test hbaseanti; then
 847     return 0
 848   fi
 849
 850   big_console_header "Checking for known anti-patterns"
 851
 852   start_clock
 853
 854   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 855   if [[ ${warnings} -gt 0 ]]; then
 856     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 857     ((result=result+1))
 858   fi
 859
 860   if [[ ${result} -gt 0 ]]; then
 861     return 1
 862   fi
 863
 864   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 865   return 0
 866 }
 867
 868 ## @description  process the javac output for generating WARNING/ERROR
 869 ## @audience     private
 870 ## @stability    evolving
 871 ## @param        input filename
 872 ## @param        output filename
 873 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
 874 # This is because that the output order of the error prone warnings is not stable, so the diff
 875 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
 876 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
 877 # really hard to sort by file name first and then line number and column number in shell...
 878 function hbase_javac_logfilter
 879 {
 880   declare input=$1
 881   declare output=$2
 882
 883   ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
 884 }
 885
 886 ## This is named so that yetus will check us right after running tests.
 887 ## Essentially, we check for normal failures and then we look for zombies.
 888 #function hbase_unit_logfilter
 889 #{
 890 #  declare testtype="unit"
 891 #  declare input=$1
 892 #  declare output=$2
 893 #  declare processes
 894 #  declare process_output
 895 #  declare zombies
 896 #  declare zombie_count=0
 897 #  declare zombie_process
 898 #
 899 #  yetus_debug "in hbase-specific unit logfilter."
 900 #
 901 #  # pass-through to whatever is counting actual failures
 902 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 903 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 904 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 905 #    "${testtype}_logfilter" "${input}" "${output}"
 906 #  fi
 907 #
 908 #  start_clock
 909 #  if [ -n "${BUILD_ID}" ]; then
 910 #    yetus_debug "Checking for zombie test processes."
 911 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 912 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 913 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 914 #      yetus_debug "${processes}"
 915 #      sleep 30
 916 #      #shellcheck disable=SC2016
 917 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 918 #        # Test our zombie still running (and that it still an hbase build item)
 919 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 920 #        if [[ -n "${process_output}" ]]; then
 921 #          yetus_error "Zombie: ${process_output}"
 922 #          ((zombie_count = zombie_count + 1))
 923 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 924 #          zombies="${zombies} ${zombie_process}"
 925 #        fi
 926 #      done
 927 #    fi
 928 #    if [ "${zombie_count}" -ne 0 ]; then
 929 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 930 #      populate_test_table "zombie unit tests" "${zombies}"
 931 #    else
 932 #      yetus_info "Zombie check complete. All test runs exited normally."
 933 #      stop_clock
 934 #    fi
 935 #  else
 936 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 937 #  fi
 938 #
 939 #}