HBASE-24118 [Flakey Tests] TestCloseRegionWhileRSCrash
[hbase.git] / dev-support / hbase-personality.sh
blobec051873b0e20f9fe02138a595b91b837ecb1e3b
1 #!/usr/bin/env bash
2 # Licensed to the Apache Software Foundation (ASF) under one or more
3 # contributor license agreements. See the NOTICE file distributed with
4 # this work for additional information regarding copyright ownership.
5 # The ASF licenses this file to You under the Apache License, Version 2.0
6 # (the "License"); you may not use this file except in compliance with
7 # the License. You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # You'll need a local installation of
18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
19 # to use this personality.
21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
22 # build from it, or use the convenience binaries provided on that download page.
24 # To run against, e.g. HBASE-15074 you'd then do
25 # ```bash
26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
27 # ```
29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
30 # ```bash
31 # test-patch --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
32 # ````
34 # pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
35 # directory / branch in order to have things match what the issue patch requests.
37 personality_plugins "all"
39 if ! declare -f "yetus_info" >/dev/null; then
41 function yetus_info
43 echo "[$(date) INFO]: $*" 1>&2
48 # work around yetus overwriting JAVA_HOME from our docker image
49 function docker_do_env_adds
51 declare k
53 for k in "${DOCKER_EXTRAENVS[@]}"; do
54 if [[ "JAVA_HOME" == "${k}" ]]; then
55 if [ -n "${JAVA_HOME}" ]; then
56 DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
58 else
59 DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
61 done
65 ## @description Globals specific to this personality
66 ## @audience private
67 ## @stability evolving
68 function personality_globals
70 BUILDTOOL=maven
71 #shellcheck disable=SC2034
72 PROJECT_NAME=hbase
73 #shellcheck disable=SC2034
74 PATCH_BRANCH_DEFAULT=master
75 #shellcheck disable=SC2034
76 JIRA_ISSUE_RE='^HBASE-[0-9]+$'
77 #shellcheck disable=SC2034
78 GITHUB_REPO="apache/hbase"
80 # TODO use PATCH_BRANCH to select jdk versions to use.
82 # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
83 # Up it. See HBASE-19902 for how we arrived at this number.
84 #shellcheck disable=SC2034
85 PROCLIMIT=10000
87 # Set docker container to run with 20g. Default is 4g in yetus.
88 # See HBASE-19902 for how we arrived at 20g.
89 #shellcheck disable=SC2034
90 DOCKERMEMLIMIT=20g
93 ## @description Parse extra arguments required by personalities, if any.
94 ## @audience private
95 ## @stability evolving
96 function personality_parse_args
98 declare i
100 for i in "$@"; do
101 case ${i} in
102 --exclude-tests-url=*)
103 delete_parameter "${i}"
104 EXCLUDE_TESTS_URL=${i#*=}
106 --include-tests-url=*)
107 delete_parameter "${i}"
108 INCLUDE_TESTS_URL=${i#*=}
110 --hadoop-profile=*)
111 delete_parameter "${i}"
112 HADOOP_PROFILE=${i#*=}
114 --skip-errorprone)
115 delete_parameter "${i}"
116 SKIP_ERRORPRONE=true
118 esac
119 done
122 ## @description Queue up modules for this personality
123 ## @audience private
124 ## @stability evolving
125 ## @param repostatus
126 ## @param testtype
127 function personality_modules
129 local repostatus=$1
130 local testtype=$2
131 local extra=""
132 local branch1jdk8=()
133 local jdk8module=""
134 local MODULES=("${CHANGED_MODULES[@]}")
136 yetus_info "Personality: ${repostatus} ${testtype}"
138 clear_personality_queue
140 extra="-DHBasePatchProcess"
141 if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
142 extra="${extra} -Dhttps.protocols=TLSv1.2"
145 if [[ -n "${HADOOP_PROFILE}" ]]; then
146 extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
149 # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
150 # full source code instead. In this case, do full compiles, tests, etc instead of per
151 # module.
152 # Used in nightly runs.
153 # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
154 # modules if root is included. HBASE-18505
155 if [[ "${BUILDMODE}" == "full" ]] || \
156 { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
157 [[ "${MODULES[*]}" =~ \. ]]; }; then
158 MODULES=(.)
161 # If the checkstyle configs change, check everything.
162 if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
163 MODULES=(.)
166 if [[ ${testtype} == mvninstall ]]; then
167 # shellcheck disable=SC2086
168 personality_enqueue_module . ${extra}
169 return
172 # This list should include any modules that require jdk8. Maven should be configured to only
173 # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
174 # module to build as yetus does if something changes in the module. Rather than try to
175 # figure out what jdk is in use so we can duplicate the module activation logic, just
176 # build at the top level if anything changes in one of these modules and let maven sort it out.
177 branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
178 if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
179 for jdk8module in "${branch1jdk8[@]}"; do
180 if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
181 MODULES=(.)
182 break
184 done
187 if [[ ${testtype} == spotbugs ]]; then
188 # Run spotbugs on each module individually to diff pre-patch and post-patch results and
189 # report new warnings for changed modules only.
190 # For some reason, spotbugs on root is not working, but running on individual modules is
191 # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
192 for module in "${CHANGED_MODULES[@]}"; do
193 # skip spotbugs on any module that lacks content in `src/main/java`
194 if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
195 -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
196 yetus_debug "no java files found under ${module}/src/main/java. skipping."
197 continue
198 else
199 # shellcheck disable=SC2086
200 personality_enqueue_module ${module} ${extra}
202 done
203 return
206 if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
207 [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
208 extra="${extra} -PerrorProne"
211 # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
212 # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
213 # tests respectively.
214 if [[ ${testtype} == unit ]]; then
215 local tests_arg=""
216 get_include_exclude_tests_arg tests_arg
217 extra="${extra} -PrunAllTests ${tests_arg}"
219 # Inject the jenkins build-id for our surefire invocations
220 # Used by zombie detection stuff, even though we're not including that yet.
221 if [ -n "${BUILD_ID}" ]; then
222 extra="${extra} -Dbuild.id=${BUILD_ID}"
225 # If the set of changed files includes CommonFSUtils then add the hbase-server
226 # module to the set of modules (if not already included) to be tested
227 for f in "${CHANGED_FILES[@]}"
229 if [[ "${f}" =~ CommonFSUtils ]]; then
230 if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ \. ]]; then
231 MODULES+=("hbase-server")
233 break
235 done
238 for module in "${MODULES[@]}"; do
239 # shellcheck disable=SC2086
240 personality_enqueue_module ${module} ${extra}
241 done
244 ## @description places where we override the built in assumptions about what tests to run
245 ## @audience private
246 ## @stability evolving
247 ## @param filename of changed file
248 function personality_file_tests
250 local filename=$1
251 yetus_debug "HBase specific personality_file_tests"
252 # If the change is to the refguide, then we don't need any builtin yetus tests
253 # the refguide test (below) will suffice for coverage.
254 if [[ ${filename} =~ src/main/asciidoc ]] ||
255 [[ ${filename} =~ src/main/xslt ]]; then
256 yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
257 else
258 # If we change our asciidoc, rebuild mvnsite
259 if [[ ${BUILDTOOL} = maven ]]; then
260 if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
261 yetus_debug "tests/mvnsite: ${filename}"
262 add_test mvnsite
265 # If we change checkstyle configs, run checkstyle
266 if [[ ${filename} =~ checkstyle.*\.xml ]]; then
267 yetus_debug "tests/checkstyle: ${filename}"
268 add_test checkstyle
270 # fallback to checking which tests based on what yetus would do by default
271 if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
272 "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
273 elif declare -f builtin_personality_file_tests >/dev/null; then
274 builtin_personality_file_tests "${filename}"
279 ## @description Uses relevant include/exclude env variable to fetch list of included/excluded
280 # tests and sets given variable to arguments to be passes to maven command.
281 ## @audience private
282 ## @stability evolving
283 ## @param name of variable to set with maven arguments
284 function get_include_exclude_tests_arg
286 local __resultvar=$1
287 yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
288 yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
289 if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
290 if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
291 excludes=$(cat excludes)
292 yetus_debug "excludes=${excludes}"
293 if [[ -n "${excludes}" ]]; then
294 eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
296 rm excludes
297 else
298 yetus_error "Wget error $? in fetching excludes file from url" \
299 "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
301 elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
302 if wget "$INCLUDE_TESTS_URL" -O "includes"; then
303 includes=$(cat includes)
304 yetus_debug "includes=${includes}"
305 if [[ -n "${includes}" ]]; then
306 eval "${__resultvar}='-Dtest=${includes}'"
308 rm includes
309 else
310 yetus_error "Wget error $? in fetching includes file from url" \
311 "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
313 else
314 # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
315 FLAKY_URL="https://builds.apache.org/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
316 if wget "${FLAKY_URL}" -O "excludes"; then
317 excludes=$(cat excludes)
318 yetus_debug "excludes=${excludes}"
319 if [[ -n "${excludes}" ]]; then
320 eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
322 rm excludes
323 else
324 yetus_error "Wget error $? in fetching excludes file from url" \
325 "${FLAKY_URL}. Ignoring and proceeding."
330 ###################################################
331 # Below here are our one-off tests specific to hbase.
332 # TODO break them into individual files so it's easier to maintain them?
334 # TODO line length check? could ignore all java files since checkstyle gets them.
336 ###################################################
338 add_test_type refguide
340 function refguide_initialize
342 maven_add_install refguide
345 function refguide_filefilter
347 local filename=$1
349 if [[ ${filename} =~ src/main/asciidoc ]] ||
350 [[ ${filename} =~ src/main/xslt ]] ||
351 [[ ${filename} =~ hbase-common/src/main/resources/hbase-default.xml ]]; then
352 add_test refguide
356 function refguide_rebuild
358 local repostatus=$1
359 local logfile="${PATCH_DIR}/${repostatus}-refguide.log"
360 declare -i count
361 declare pdf_output
363 if ! verify_needed_test refguide; then
364 return 0
367 big_console_header "Checking we can create the ref guide on ${repostatus}"
369 start_clock
371 # disabled because "maven_executor" needs to return both command and args
372 # shellcheck disable=2046
373 echo_and_redirect "${logfile}" \
374 $(maven_executor) clean site --batch-mode \
375 -pl . \
376 -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
377 -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
379 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
380 if [[ ${count} -gt 0 ]]; then
381 add_vote_table -1 refguide "${repostatus} has ${count} errors when building the reference guide."
382 add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
383 return 1
386 if ! mv target/site "${PATCH_DIR}/${repostatus}-site"; then
387 add_vote_table -1 refguide "${repostatus} failed to produce a site directory."
388 add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
389 return 1
392 if [[ ! -f "${PATCH_DIR}/${repostatus}-site/book.html" ]]; then
393 add_vote_table -1 refguide "${repostatus} failed to produce the html version of the reference guide."
394 add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
395 return 1
398 if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
399 pdf_output="book.pdf"
400 else
401 pdf_output="apache_hbase_reference_guide.pdf"
404 if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
405 add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
406 add_footer_table refguide "@@BASE@@/${repostatus}-refguide.log"
407 return 1
410 add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
411 add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
412 return 0
415 add_test_type shadedjars
418 function shadedjars_initialize
420 yetus_debug "initializing shaded client checks."
421 maven_add_install shadedjars
424 ## @description only run the test if java changes.
425 ## @audience private
426 ## @stability evolving
427 ## @param filename
428 function shadedjars_filefilter
430 local filename=$1
432 if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
433 add_test shadedjars
437 ## @description test the shaded client artifacts
438 ## @audience private
439 ## @stability evolving
440 ## @param repostatus
441 function shadedjars_rebuild
443 local repostatus=$1
444 local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
446 if ! verify_needed_test shadedjars; then
447 return 0
450 big_console_header "Checking shaded client builds on ${repostatus}"
452 start_clock
454 local -a maven_args=('clean' 'verify' '-fae' '--batch-mode'
455 '-pl' 'hbase-shaded/hbase-shaded-check-invariants' '-am'
456 '-Dtest=NoUnitTests' '-DHBasePatchProcess' '-Prelease'
457 '-Dmaven.javadoc.skip=true' '-Dcheckstyle.skip=true' '-Dspotbugs.skip=true')
458 if [[ -n "${HADOOP_PROFILE}" ]]; then
459 maven_args+=("-Dhadoop.profile=${HADOOP_PROFILE}")
462 # disabled because "maven_executor" needs to return both command and args
463 # shellcheck disable=2046
464 echo_and_redirect "${logfile}" $(maven_executor) "${maven_args[@]}"
466 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
467 if [[ ${count} -gt 0 ]]; then
468 add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
469 add_footer_table shadedjars "@@BASE@@/${repostatus}-shadedjars.txt"
470 return 1
473 add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
474 return 0
477 ###################################################
479 add_test_type hadoopcheck
481 ## @description hadoopcheck file filter
482 ## @audience private
483 ## @stability evolving
484 ## @param filename
485 function hadoopcheck_filefilter
487 local filename=$1
489 if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
490 add_test hadoopcheck
494 ## @description Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
495 ## @audience private
496 ## @stability evolving
497 function hadoopcheck_parse_args
499 declare i
501 for i in "$@"; do
502 case ${i} in
503 --quick-hadoopcheck)
504 delete_parameter "${i}"
505 QUICK_HADOOPCHECK=true
507 esac
508 done
511 ## @description Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
512 ## @audience private
513 ## @stability evolving
514 function hadoopcheck_docker_support
516 DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
519 ## @description hadoopcheck test
520 ## @audience private
521 ## @stability evolving
522 ## @param repostatus
523 function hadoopcheck_rebuild
525 local repostatus=$1
526 local hadoopver
527 local logfile
528 local count
529 local result=0
530 local hbase_hadoop2_versions
531 local hbase_hadoop3_versions
533 if [[ "${repostatus}" = branch ]]; then
534 return 0
537 if ! verify_needed_test hadoopcheck; then
538 return 0
541 big_console_header "Compiling against various Hadoop versions"
543 start_clock
545 # All supported Hadoop versions that we want to test the compilation with
546 # See the Hadoop section on prereqs in the HBase Reference Guide
547 if [[ "${PATCH_BRANCH}" = branch-1.3 ]]; then
548 yetus_info "Setting Hadoop 2 versions to test based on branch-1.3 rules."
549 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
550 hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
551 else
552 hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
554 elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
555 yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
556 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
557 hbase_hadoop2_versions="2.7.7"
558 else
559 hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
561 elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
562 yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
563 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
564 hbase_hadoop2_versions="2.8.5 2.9.2"
565 else
566 hbase_hadoop2_versions="2.8.5 2.9.2"
568 elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
569 yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
570 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
571 hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
572 else
573 hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
575 elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
576 yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
577 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
578 hbase_hadoop2_versions="2.7.7 2.8.5"
579 else
580 hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
582 elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
583 yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
584 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
585 hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
586 else
587 hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
589 else
590 yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+/master/feature branch rules."
591 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
592 hbase_hadoop2_versions="2.10.0"
593 else
594 hbase_hadoop2_versions="2.10.0"
597 if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
598 yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
599 hbase_hadoop3_versions=""
600 elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
601 yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
602 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
603 hbase_hadoop3_versions="3.0.3 3.1.2"
604 else
605 hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
607 else
608 yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
609 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
610 hbase_hadoop3_versions="3.1.2"
611 else
612 hbase_hadoop3_versions="3.1.1 3.1.2"
616 export MAVEN_OPTS="${MAVEN_OPTS}"
617 for hadoopver in ${hbase_hadoop2_versions}; do
618 logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
619 # disabled because "maven_executor" needs to return both command and args
620 # shellcheck disable=2046
621 echo_and_redirect "${logfile}" \
622 $(maven_executor) clean install \
623 -DskipTests -DHBasePatchProcess \
624 -Dhadoop-two.version="${hadoopver}"
625 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
626 if [[ ${count} -gt 0 ]]; then
627 add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
628 add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
629 ((result=result+1))
631 done
633 for hadoopver in ${hbase_hadoop3_versions}; do
634 logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
635 # disabled because "maven_executor" needs to return both command and args
636 # shellcheck disable=2046
637 echo_and_redirect "${logfile}" \
638 $(maven_executor) clean install \
639 -DskipTests -DHBasePatchProcess \
640 -Dhadoop-three.version="${hadoopver}" \
641 -Dhadoop.profile=3.0
642 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
643 if [[ ${count} -gt 0 ]]; then
644 add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
645 add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
646 ((result=result+1))
648 done
650 if [[ ${result} -gt 0 ]]; then
651 return 1
654 if [[ -n "${hbase_hadoop3_versions}" ]]; then
655 add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
656 else
657 add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
660 logfile="${PATCH_DIR}/patch-install-after-hadoopcheck.txt"
661 echo_and_redirect "${logfile}" \
662 $(maven_executor) clean install \
663 -DskipTests -DHBasePatchProcess
665 return 0
668 ######################################
670 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
671 add_test_type hbaseprotoc
673 ## @description hbaseprotoc file filter
674 ## @audience private
675 ## @stability evolving
676 ## @param filename
677 function hbaseprotoc_filefilter
679 local filename=$1
681 if [[ ${filename} =~ \.proto$ ]]; then
682 add_test hbaseprotoc
686 ## @description check hbase proto compilation
687 ## @audience private
688 ## @stability evolving
689 ## @param repostatus
690 function hbaseprotoc_rebuild
692 declare repostatus=$1
693 declare i=0
694 declare fn
695 declare module
696 declare logfile
697 declare count
698 declare result
700 if [[ "${repostatus}" = branch ]]; then
701 return 0
704 if ! verify_needed_test hbaseprotoc; then
705 return 0
708 big_console_header "HBase protoc plugin: ${BUILDMODE}"
710 start_clock
712 personality_modules patch hbaseprotoc
713 # Need to run 'install' instead of 'compile' because shading plugin
714 # is hooked-up to 'install'; else hbase-protocol-shaded is left with
715 # half of its process done.
716 modules_workers patch hbaseprotoc install -DskipTests -X -DHBasePatchProcess
718 # shellcheck disable=SC2153
719 until [[ $i -eq "${#MODULE[@]}" ]]; do
720 if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
721 ((result=result+1))
722 ((i=i+1))
723 continue
725 module=${MODULE[$i]}
726 fn=$(module_file_fragment "${module}")
727 logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
729 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
731 if [[ ${count} -gt 0 ]]; then
732 module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
733 "${count} new protoc errors in ${module}."
734 ((result=result+1))
736 ((i=i+1))
737 done
739 modules_messages patch hbaseprotoc true
740 if [[ ${result} -gt 0 ]]; then
741 return 1
743 return 0
746 ######################################
748 add_test_type hbaseanti
750 ## @description hbaseanti file filter
751 ## @audience private
752 ## @stability evolving
753 ## @param filename
754 function hbaseanti_filefilter
756 local filename=$1
758 if [[ ${filename} =~ \.java$ ]]; then
759 add_test hbaseanti
763 ## @description hbaseanti patch file check
764 ## @audience private
765 ## @stability evolving
766 ## @param filename
767 function hbaseanti_patchfile
769 local patchfile=$1
770 local warnings
771 local result
773 if [[ "${BUILDMODE}" = full ]]; then
774 return 0
777 if ! verify_needed_test hbaseanti; then
778 return 0
781 big_console_header "Checking for known anti-patterns"
783 start_clock
785 warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
786 if [[ ${warnings} -gt 0 ]]; then
787 add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
788 ((result=result+1))
791 if [[ ${result} -gt 0 ]]; then
792 return 1
795 add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
796 return 0
799 ## @description process the javac output for generating WARNING/ERROR
800 ## @audience private
801 ## @stability evolving
802 ## @param input filename
803 ## @param output filename
804 # Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
805 # This is because that the output order of the error prone warnings is not stable, so the diff
806 # method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
807 # line number being sorted by lexicographical so the output maybe a bit strange to human but it is
808 # really hard to sort by file name first and then line number and column number in shell...
809 function hbase_javac_logfilter
811 declare input=$1
812 declare output=$2
814 ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
817 ## This is named so that yetus will check us right after running tests.
818 ## Essentially, we check for normal failures and then we look for zombies.
819 #function hbase_unit_logfilter
821 # declare testtype="unit"
822 # declare input=$1
823 # declare output=$2
824 # declare processes
825 # declare process_output
826 # declare zombies
827 # declare zombie_count=0
828 # declare zombie_process
830 # yetus_debug "in hbase-specific unit logfilter."
832 # # pass-through to whatever is counting actual failures
833 # if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
834 # "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
835 # elif declare -f ${testtype}_logfilter >/dev/null; then
836 # "${testtype}_logfilter" "${input}" "${output}"
837 # fi
839 # start_clock
840 # if [ -n "${BUILD_ID}" ]; then
841 # yetus_debug "Checking for zombie test processes."
842 # processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
843 # if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
844 # yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
845 # yetus_debug "${processes}"
846 # sleep 30
847 # #shellcheck disable=SC2016
848 # for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
849 # # Test our zombie still running (and that it still an hbase build item)
850 # process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
851 # if [[ -n "${process_output}" ]]; then
852 # yetus_error "Zombie: ${process_output}"
853 # ((zombie_count = zombie_count + 1))
854 # zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
855 # zombies="${zombies} ${zombie_process}"
856 # fi
857 # done
858 # fi
859 # if [ "${zombie_count}" -ne 0 ]; then
860 # add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
861 # populate_test_table "zombie unit tests" "${zombies}"
862 # else
863 # yetus_info "Zombie check complete. All test runs exited normally."
864 # stop_clock
865 # fi
866 # else
867 # add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
868 # fi