dev-support/hbase-personality.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one or more
   3 # contributor license agreements.  See the NOTICE file distributed with
   4 # this work for additional information regarding copyright ownership.
   5 # The ASF licenses this file to You under the Apache License, Version 2.0
   6 # (the "License"); you may not use this file except in compliance with
   7 # the License.  You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # You'll need a local installation of
  18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
  19 # to use this personality.
  20 #
  21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
  22 # build from it, or use the convenience binaries provided on that download page.
  23 #
  24 # To run against, e.g. HBASE-15074 you'd then do
  25 # ```bash
  26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
  27 # ```
  28 #
  29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
  30 # ```bash
  31 # test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
  32 # ````
  33 #
  34 # pass the `--jenkins` flag if you want to allow test-patch to destructively alter local working
  35 # directory / branch in order to have things match what the issue patch requests.
  36
  37 personality_plugins "all"
  38
  39 if ! declare -f "yetus_info" >/dev/null; then
  40
  41   function yetus_info
  42   {
  43     echo "[$(date) INFO]: $*" 1>&2
  44   }
  45
  46 fi
  47
  48 ## @description  Globals specific to this personality
  49 ## @audience     private
  50 ## @stability    evolving
  51 function personality_globals
  52 {
  53   BUILDTOOL=maven
  54   #shellcheck disable=SC2034
  55   PROJECT_NAME=hbase
  56   #shellcheck disable=SC2034
  57   PATCH_BRANCH_DEFAULT=master
  58   #shellcheck disable=SC2034
  59   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
  60   #shellcheck disable=SC2034
  61   GITHUB_REPO="apache/hbase"
  62
  63   # TODO use PATCH_BRANCH to select jdk versions to use.
  64
  65   # Override the maven options
  66   MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
  67
  68   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
  69   # Up it. See HBASE-19902 for how we arrived at this number.
  70   #shellcheck disable=SC2034
  71   PROCLIMIT=10000
  72
  73   # Set docker container to run with 20g. Default is 4g in yetus.
  74   # See HBASE-19902 for how we arrived at 20g.
  75   #shellcheck disable=SC2034
  76   DOCKERMEMLIMIT=20g
  77 }
  78
  79 ## @description  Parse extra arguments required by personalities, if any.
  80 ## @audience     private
  81 ## @stability    evolving
  82 function personality_parse_args
  83 {
  84   declare i
  85
  86   for i in "$@"; do
  87     case ${i} in
  88       --exclude-tests-url=*)
  89         EXCLUDE_TESTS_URL=${i#*=}
  90       ;;
  91       --include-tests-url=*)
  92         INCLUDE_TESTS_URL=${i#*=}
  93       ;;
  94       --hadoop-profile=*)
  95         HADOOP_PROFILE=${i#*=}
  96       ;;
  97     esac
  98   done
  99 }
 100
 101 ## @description  Queue up modules for this personality
 102 ## @audience     private
 103 ## @stability    evolving
 104 ## @param        repostatus
 105 ## @param        testtype
 106 function personality_modules
 107 {
 108   local repostatus=$1
 109   local testtype=$2
 110   local extra=""
 111   local MODULES=("${CHANGED_MODULES[@]}")
 112
 113   yetus_info "Personality: ${repostatus} ${testtype}"
 114
 115   clear_personality_queue
 116
 117   extra="-DHBasePatchProcess"
 118   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 119     extra="${extra} -Dhttps.protocols=TLSv1.2"
 120   fi
 121
 122   if [[ -n "${HADOOP_PROFILE}" ]]; then
 123     extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
 124   fi
 125
 126   # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
 127   # full source code instead. In this case, do full compiles, tests, etc instead of per
 128   # module.
 129   # Used in nightly runs.
 130   # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
 131   # modules if root is included. HBASE-18505
 132   if [[ "${BUILDMODE}" == "full" ]] || \
 133      ( ( [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]] ) && \
 134      [[ "${MODULES[*]}" =~ \. ]] ); then
 135     MODULES=(.)
 136   fi
 137
 138   # If the checkstyle configs change, check everything.
 139   if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
 140     MODULES=(.)
 141   fi
 142
 143   if [[ ${testtype} == mvninstall ]]; then
 144     # shellcheck disable=SC2086
 145     personality_enqueue_module . ${extra}
 146     return
 147   fi
 148
 149   if [[ ${testtype} == findbugs ]]; then
 150     # Run findbugs on each module individually to diff pre-patch and post-patch results and
 151     # report new warnings for changed modules only.
 152     # For some reason, findbugs on root is not working, but running on individual modules is
 153     # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
 154     for module in "${CHANGED_MODULES[@]}"; do
 155       # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
 156       # in src/main/java where findbugs goes to look
 157       if [[ ${module} == hbase-shell ]]; then
 158         continue
 159       elif [[ ${module} == hbase-it ]]; then
 160         continue
 161       else
 162         # shellcheck disable=SC2086
 163         personality_enqueue_module ${module} ${extra}
 164       fi
 165     done
 166     return
 167   fi
 168
 169   if [[ ${testtype} == compile ]]; then
 170     extra="${extra} -PerrorProne"
 171   fi
 172
 173   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
 174   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
 175   # tests respectively.
 176   if [[ ${testtype} == unit ]]; then
 177     local tests_arg=""
 178     get_include_exclude_tests_arg tests_arg
 179     extra="${extra} -PrunAllTests ${tests_arg}"
 180
 181     # Inject the jenkins build-id for our surefire invocations
 182     # Used by zombie detection stuff, even though we're not including that yet.
 183     if [ -n "${BUILD_ID}" ]; then
 184       extra="${extra} -Dbuild.id=${BUILD_ID}"
 185     fi
 186
 187     # If the set of changed files includes CommonFSUtils then add the hbase-server
 188     # module to the set of modules (if not already included) to be tested
 189     for f in "${CHANGED_FILES[@]}"
 190     do
 191       if [[ "${f}" =~ CommonFSUtils ]]; then
 192         if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
 193           MODULES+=("hbase-server")
 194         fi
 195         break
 196       fi
 197     done
 198   fi
 199
 200   for module in "${MODULES[@]}"; do
 201     # shellcheck disable=SC2086
 202     personality_enqueue_module ${module} ${extra}
 203   done
 204 }
 205
 206 ## @description places where we override the built in assumptions about what tests to run
 207 ## @audience    private
 208 ## @stability   evolving
 209 ## @param       filename of changed file
 210 function personality_file_tests
 211 {
 212   local filename=$1
 213   yetus_debug "HBase specific personality_file_tests"
 214   # If the change is to the refguide, then we don't need any builtin yetus tests
 215   # the refguide test (below) will suffice for coverage.
 216   if [[ ${filename} =~ src/main/asciidoc ]] ||
 217      [[ ${filename} =~ src/main/xslt ]]; then
 218     yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
 219   else
 220     # If we change our asciidoc, rebuild mvnsite
 221     if [[ ${BUILDTOOL} = maven ]]; then
 222       if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
 223         yetus_debug "tests/mvnsite: ${filename}"
 224         add_test mvnsite
 225       fi
 226     fi
 227     # If we change checkstyle configs, run checkstyle
 228     if [[ ${filename} =~ checkstyle.*\.xml ]]; then
 229       yetus_debug "tests/checkstyle: ${filename}"
 230       add_test checkstyle
 231     fi
 232     # fallback to checking which tests based on what yetus would do by default
 233     if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
 234       "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
 235     elif declare -f builtin_personality_file_tests >/dev/null; then
 236       builtin_personality_file_tests "${filename}"
 237     fi
 238   fi
 239 }
 240
 241 ## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
 242 #                tests and sets given variable to arguments to be passes to maven command.
 243 ## @audience     private
 244 ## @stability    evolving
 245 ## @param        name of variable to set with maven arguments
 246 function get_include_exclude_tests_arg
 247 {
 248   local  __resultvar=$1
 249   yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
 250   yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
 251   if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 252       if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
 253         excludes=$(cat excludes)
 254         yetus_debug "excludes=${excludes}"
 255         if [[ -n "${excludes}" ]]; then
 256           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 257         fi
 258         rm excludes
 259       else
 260         yetus_error "Wget error $? in fetching excludes file from url" \
 261              "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
 262       fi
 263   elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
 264       if wget "$INCLUDE_TESTS_URL" -O "includes"; then
 265         includes=$(cat includes)
 266         yetus_debug "includes=${includes}"
 267         if [[ -n "${includes}" ]]; then
 268           eval "${__resultvar}='-Dtest=${includes}'"
 269         fi
 270         rm includes
 271       else
 272         yetus_error "Wget error $? in fetching includes file from url" \
 273              "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
 274       fi
 275   else
 276     # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
 277     FLAKY_URL="https://builds.apache.org/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
 278     if wget "${FLAKY_URL}" -O "excludes"; then
 279       excludes=$(cat excludes)
 280         yetus_debug "excludes=${excludes}"
 281         if [[ -n "${excludes}" ]]; then
 282           eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
 283         fi
 284         rm excludes
 285       else
 286         yetus_error "Wget error $? in fetching excludes file from url" \
 287              "${FLAKY_URL}. Ignoring and proceeding."
 288       fi
 289   fi
 290 }
 291
 292 ###################################################
 293 # Below here are our one-off tests specific to hbase.
 294 # TODO break them into individual files so it's easier to maintain them?
 295
 296 # TODO line length check? could ignore all java files since checkstyle gets them.
 297
 298 ###################################################
 299
 300 add_test_type refguide
 301
 302 function refguide_initialize
 303 {
 304   maven_add_install refguide
 305 }
 306
 307 function refguide_filefilter
 308 {
 309   local filename=$1
 310
 311   if [[ ${filename} =~ src/main/asciidoc ]] ||
 312      [[ ${filename} =~ src/main/xslt ]] ||
 313      [[ ${filename} =~ hbase-common/src/main/resources/hbase-default.xml ]]; then
 314     add_test refguide
 315   fi
 316 }
 317
 318 function refguide_rebuild
 319 {
 320   local repostatus=$1
 321   local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
 322   declare -i count
 323   declare pdf_output
 324
 325   if ! verify_needed_test refguide; then
 326     return 0
 327   fi
 328
 329   big_console_header "Checking we can create the ref guide on ${repostatus}"
 330
 331   start_clock
 332
 333   # disabled because "maven_executor" needs to return both command and args
 334   # shellcheck disable=2046
 335   echo_and_redirect "${logfile}" \
 336     $(maven_executor) clean site --batch-mode \
 337       -pl . \
 338       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 339       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 340
 341   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 342   if [[ ${count} -gt 0 ]]; then
 343     add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
 344     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 345     return 1
 346   fi
 347
 348   if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
 349     add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
 350     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 351     return 1
 352   fi
 353
 354   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
 355     add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
 356     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 357     return 1
 358   fi
 359
 360   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 361     pdf_output="book.pdf"
 362   else
 363     pdf_output="apache_hbase_reference_guide.pdf"
 364   fi
 365
 366   if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
 367     add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
 368     add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
 369     return 1
 370   fi
 371
 372   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
 373   add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
 374   return 0
 375 }
 376
 377 add_test_type shadedjars
 378
 379
 380 function shadedjars_initialize
 381 {
 382   yetus_debug "initializing shaded client checks."
 383   maven_add_install shadedjars
 384 }
 385
 386 ## @description  only run the test if java changes.
 387 ## @audience     private
 388 ## @stability    evolving
 389 ## @param        filename
 390 function shadedjars_filefilter
 391 {
 392   local filename=$1
 393
 394   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 395     add_test shadedjars
 396   fi
 397 }
 398
 399 ## @description test the shaded client artifacts
 400 ## @audience private
 401 ## @stability evolving
 402 ## @param repostatus
 403 function shadedjars_rebuild
 404 {
 405   local repostatus=$1
 406   local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
 407
 408   if ! verify_needed_test shadedjars; then
 409     return 0
 410   fi
 411
 412   big_console_header "Checking shaded client builds on ${repostatus}"
 413
 414   start_clock
 415
 416   # disabled because "maven_executor" needs to return both command and args
 417   # shellcheck disable=2046
 418   echo_and_redirect "${logfile}" \
 419     $(maven_executor) clean verify -fae --batch-mode \
 420       -pl hbase-shaded/hbase-shaded-check-invariants -am \
 421       -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
 422       -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
 423
 424   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 425   if [[ ${count} -gt 0 ]]; then
 426     add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
 427     add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
 428     return 1
 429   fi
 430
 431   add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
 432   return 0
 433 }
 434
 435 ###################################################
 436
 437 add_test_type hadoopcheck
 438
 439 ## @description  hadoopcheck file filter
 440 ## @audience     private
 441 ## @stability    evolving
 442 ## @param        filename
 443 function hadoopcheck_filefilter
 444 {
 445   local filename=$1
 446
 447   if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
 448     add_test hadoopcheck
 449   fi
 450 }
 451
 452 ## @description  Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
 453 ## @audience     private
 454 ## @stability    evolving
 455 function hadoopcheck_parse_args
 456 {
 457   declare i
 458
 459   for i in "$@"; do
 460     case ${i} in
 461       --quick-hadoopcheck)
 462         QUICK_HADOOPCHECK=true
 463       ;;
 464     esac
 465   done
 466 }
 467
 468 ## @description  Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
 469 ## @audience     private
 470 ## @stability    evolving
 471 function hadoopcheck_docker_support
 472 {
 473   DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
 474 }
 475
 476 ## @description  hadoopcheck test
 477 ## @audience     private
 478 ## @stability    evolving
 479 ## @param        repostatus
 480 function hadoopcheck_rebuild
 481 {
 482   local repostatus=$1
 483   local hadoopver
 484   local logfile
 485   local count
 486   local result=0
 487   local hbase_hadoop2_versions
 488   local hbase_hadoop3_versions
 489
 490   if [[ "${repostatus}" = branch ]]; then
 491     return 0
 492   fi
 493
 494   if ! verify_needed_test hadoopcheck; then
 495     return 0
 496   fi
 497
 498   big_console_header "Compiling against various Hadoop versions"
 499
 500   start_clock
 501
 502   # All supported Hadoop versions that we want to test the compilation with
 503   # See the Hadoop section on prereqs in the HBase Reference Guide
 504   hbase_common_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4"
 505   if [[ "${PATCH_BRANCH}" = branch-1.* ]] && [[ "${PATCH_BRANCH#branch-1.}" -lt "5" ]]; then
 506     yetus_info "Setting Hadoop 2 versions to test based on before-branch-1.5 rules."
 507     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 508       hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.4"
 509     else
 510       hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 ${hbase_common_hadoop2_versions}"
 511     fi
 512   elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
 513     yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
 514     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 515       hbase_hadoop2_versions="2.6.5 2.7.4"
 516     else
 517       hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 ${hbase_common_hadoop2_versions}"
 518     fi
 519   else
 520     yetus_info "Setting Hadoop 2 versions to test based on branch-1.5+/branch-2.1+/master/feature branch rules."
 521     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
 522       hbase_hadoop2_versions="2.7.4"
 523     else
 524       hbase_hadoop2_versions="${hbase_common_hadoop2_versions}"
 525     fi
 526   fi
 527   hbase_hadoop3_versions="3.0.0"
 528   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
 529     hbase_hadoop3_versions=""
 530   fi
 531
 532   export MAVEN_OPTS="${MAVEN_OPTS}"
 533   for hadoopver in ${hbase_hadoop2_versions}; do
 534     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 535     # disabled because "maven_executor" needs to return both command and args
 536     # shellcheck disable=2046
 537     echo_and_redirect "${logfile}" \
 538       $(maven_executor) clean install \
 539         -DskipTests -DHBasePatchProcess \
 540         -Dhadoop-two.version="${hadoopver}"
 541     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 542     if [[ ${count} -gt 0 ]]; then
 543       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 544       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 545       ((result=result+1))
 546     fi
 547   done
 548
 549   for hadoopver in ${hbase_hadoop3_versions}; do
 550     logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
 551     # disabled because "maven_executor" needs to return both command and args
 552     # shellcheck disable=2046
 553     echo_and_redirect "${logfile}" \
 554       $(maven_executor) clean install \
 555         -DskipTests -DHBasePatchProcess \
 556         -Dhadoop-three.version="${hadoopver}" \
 557         -Dhadoop.profile=3.0
 558     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 559     if [[ ${count} -gt 0 ]]; then
 560       add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
 561       add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
 562       ((result=result+1))
 563     fi
 564   done
 565
 566   if [[ ${result} -gt 0 ]]; then
 567     return 1
 568   fi
 569
 570   if [[ -n "${hbase_hadoop3_versions}" ]]; then
 571     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
 572   else
 573     add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
 574   fi
 575   return 0
 576 }
 577
 578 ######################################
 579
 580 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
 581 add_test_type hbaseprotoc
 582
 583 ## @description  hbaseprotoc file filter
 584 ## @audience     private
 585 ## @stability    evolving
 586 ## @param        filename
 587 function hbaseprotoc_filefilter
 588 {
 589   local filename=$1
 590
 591   if [[ ${filename} =~ \.proto$ ]]; then
 592     add_test hbaseprotoc
 593   fi
 594 }
 595
 596 ## @description  check hbase proto compilation
 597 ## @audience     private
 598 ## @stability    evolving
 599 ## @param        repostatus
 600 function hbaseprotoc_rebuild
 601 {
 602   declare repostatus=$1
 603   declare i=0
 604   declare fn
 605   declare module
 606   declare logfile
 607   declare count
 608   declare result
 609
 610   if [[ "${repostatus}" = branch ]]; then
 611     return 0
 612   fi
 613
 614   if ! verify_needed_test hbaseprotoc; then
 615     return 0
 616   fi
 617
 618   big_console_header "HBase protoc plugin: ${BUILDMODE}"
 619
 620   start_clock
 621
 622   personality_modules patch hbaseprotoc
 623   # Need to run 'install' instead of 'compile' because shading plugin
 624   # is hooked-up to 'install'; else hbase-protocol-shaded is left with
 625   # half of its process done.
 626   modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
 627
 628   # shellcheck disable=SC2153
 629   until [[ $i -eq "${#MODULE[@]}" ]]; do
 630     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
 631       ((result=result+1))
 632       ((i=i+1))
 633       continue
 634     fi
 635     module=${MODULE[$i]}
 636     fn=$(module_file_fragment "${module}")
 637     logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
 638
 639     count=$(${GREP} -c '\[ERROR\]' "${logfile}")
 640
 641     if [[ ${count} -gt 0 ]]; then
 642       module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
 643         "${count} new protoc errors in ${module}."
 644       ((result=result+1))
 645     fi
 646     ((i=i+1))
 647   done
 648
 649   modules_messages patch hbaseprotoc true
 650   if [[ ${result} -gt 0 ]]; then
 651     return 1
 652   fi
 653   return 0
 654 }
 655
 656 ######################################
 657
 658 add_test_type hbaseanti
 659
 660 ## @description  hbaseanti file filter
 661 ## @audience     private
 662 ## @stability    evolving
 663 ## @param        filename
 664 function hbaseanti_filefilter
 665 {
 666   local filename=$1
 667
 668   if [[ ${filename} =~ \.java$ ]]; then
 669     add_test hbaseanti
 670   fi
 671 }
 672
 673 ## @description  hbaseanti patch file check
 674 ## @audience     private
 675 ## @stability    evolving
 676 ## @param        filename
 677 function hbaseanti_patchfile
 678 {
 679   local patchfile=$1
 680   local warnings
 681   local result
 682
 683   if [[ "${BUILDMODE}" = full ]]; then
 684     return 0
 685   fi
 686
 687   if ! verify_needed_test hbaseanti; then
 688     return 0
 689   fi
 690
 691   big_console_header "Checking for known anti-patterns"
 692
 693   start_clock
 694
 695   warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
 696   if [[ ${warnings} -gt 0 ]]; then
 697     add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
 698     ((result=result+1))
 699   fi
 700
 701   if [[ ${result} -gt 0 ]]; then
 702     return 1
 703   fi
 704
 705   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
 706   return 0
 707 }
 708
 709 ## This is named so that yetus will check us right after running tests.
 710 ## Essentially, we check for normal failures and then we look for zombies.
 711 #function hbase_unit_logfilter
 712 #{
 713 #  declare testtype="unit"
 714 #  declare input=$1
 715 #  declare output=$2
 716 #  declare processes
 717 #  declare process_output
 718 #  declare zombies
 719 #  declare zombie_count=0
 720 #  declare zombie_process
 721 #
 722 #  yetus_debug "in hbase-specific unit logfilter."
 723 #
 724 #  # pass-through to whatever is counting actual failures
 725 #  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
 726 #    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
 727 #  elif declare -f ${testtype}_logfilter >/dev/null; then
 728 #    "${testtype}_logfilter" "${input}" "${output}"
 729 #  fi
 730 #
 731 #  start_clock
 732 #  if [ -n "${BUILD_ID}" ]; then
 733 #    yetus_debug "Checking for zombie test processes."
 734 #    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 735 #    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
 736 #      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
 737 #      yetus_debug "${processes}"
 738 #      sleep 30
 739 #      #shellcheck disable=SC2016
 740 #      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
 741 #        # Test our zombie still running (and that it still an hbase build item)
 742 #        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
 743 #        if [[ -n "${process_output}" ]]; then
 744 #          yetus_error "Zombie: ${process_output}"
 745 #          ((zombie_count = zombie_count + 1))
 746 #          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
 747 #          zombies="${zombies} ${zombie_process}"
 748 #        fi
 749 #      done
 750 #    fi
 751 #    if [ "${zombie_count}" -ne 0 ]; then
 752 #      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
 753 #      populate_test_table "zombie unit tests" "${zombies}"
 754 #    else
 755 #      yetus_info "Zombie check complete. All test runs exited normally."
 756 #      stop_clock
 757 #    fi
 758 #  else
 759 #    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
 760 #  fi
 761 #
 762 #}