dev-support/jenkins_precommit_github_yetus.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e
  20
  21 # place ourselves in the directory containing the hbase and yetus checkouts
  22 cd "$(dirname "$0")/../.."
  23 echo "executing from $(pwd)"
  24
  25 if [[ "true" = "${DEBUG}" ]]; then
  26   set -x
  27   printenv 2>&1 | sort
  28 fi
  29
  30 declare -i missing_env=0
  31 declare -a required_envs=(
  32   # these ENV variables define the required API with Jenkinsfile_GitHub
  33   "ARCHIVE_PATTERN_LIST"
  34   "BUILD_URL_ARTIFACTS"
  35   "DOCKERFILE"
  36   "GITHUB_PASSWORD"
  37   "GITHUB_USER"
  38   "PATCHDIR"
  39   "PLUGINS"
  40   "SET_JAVA_HOME"
  41   "SOURCEDIR"
  42   "TESTS_FILTER"
  43   "YETUSDIR"
  44 )
  45 # Validate params
  46 for required_env in "${required_envs[@]}"; do
  47   if [ -z "${!required_env}" ]; then
  48     echo "[ERROR] Required environment variable '${required_env}' is not set."
  49     missing_env=${missing_env}+1
  50   fi
  51 done
  52
  53 if [ ${missing_env} -gt 0 ]; then
  54   echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
  55        "on Jenkins, then please file a JIRA about the error."
  56   exit 1
  57 fi
  58
  59 # TODO (HBASE-23900): cannot assume test-patch runs directly from sources
  60 TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
  61
  62 # this must be clean for every run
  63 rm -rf "${PATCHDIR}"
  64 mkdir -p "${PATCHDIR}"
  65
  66 # Checking on H* machine nonsense
  67 mkdir "${PATCHDIR}/machine"
  68 "${SOURCEDIR}/dev-support/gather_machine_environment.sh" "${PATCHDIR}/machine"
  69
  70 # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
  71 # Otherwise exit, because we don't want HBase to do a
  72 # full build.  We wouldn't normally do this check for smaller
  73 # projects. :)
  74 if [[ -z "${CHANGE_URL}" ]]; then
  75   echo "Full build skipped" > "${PATCHDIR}/report.html"
  76   exit 0
  77 fi
  78 # enable debug output for yetus
  79 if [[ "true" = "${DEBUG}" ]]; then
  80   YETUS_ARGS+=("--debug")
  81 fi
  82 # If we're doing docker, make sure we don't accidentally pollute the image with a host java path
  83 if [ -n "${JAVA_HOME}" ]; then
  84   unset JAVA_HOME
  85 fi
  86 YETUS_ARGS+=("--ignore-unknown-options=true")
  87 YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
  88 # where the source is located
  89 YETUS_ARGS+=("--basedir=${SOURCEDIR}")
  90 # our project defaults come from a personality file
  91 # which will get loaded automatically by setting the project name
  92 YETUS_ARGS+=("--project=hbase")
  93 # lots of different output formats
  94 YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
  95 YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
  96 YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
  97 # enable writing back to Github
  98 YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
  99 YETUS_ARGS+=("--github-user=${GITHUB_USER}")
 100 # auto-kill any surefire stragglers during unit test runs
 101 YETUS_ARGS+=("--reapermode=kill")
 102 # set relatively high limits for ASF machines
 103 # changing these to higher values may cause problems
 104 # with other jobs on systemd-enabled machines
 105 YETUS_ARGS+=("--dockermemlimit=20g")
 106 # -1 spotbugs issues that show up prior to the patch being applied
 107 YETUS_ARGS+=("--spotbugs-strict-precheck")
 108 # rsync these files back into the archive dir
 109 YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
 110 # URL for user-side presentation in reports and such to our artifacts
 111 YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}")
 112 # plugins to enable
 113 YETUS_ARGS+=("--plugins=${PLUGINS},-findbugs")
 114 # run in docker mode and specifically point to our
 115 # Dockerfile since we don't want to use the auto-pulled version.
 116 YETUS_ARGS+=("--docker")
 117 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 118 YETUS_ARGS+=("--mvn-custom-repos")
 119 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
 120 YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
 121 YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
 122 YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
 123 YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 124 YETUS_ARGS+=("--quick-hadoopcheck")
 125 if [[ "${SKIP_ERRORPRONE}" = "true" ]]; then
 126   # skip error prone
 127   YETUS_ARGS+=("--skip-errorprone")
 128 fi
 129 # effectively treat dev-support as a custom maven module
 130 YETUS_ARGS+=("--skip-dirs=dev-support")
 131 # For testing with specific hadoop version. Activates corresponding profile in maven runs.
 132 if [[ -n "${HADOOP_PROFILE}" ]]; then
 133   # Master has only Hadoop3 support. We don't need to activate any profile.
 134   # The Jenkinsfile should not attempt to run any Hadoop2 tests.
 135   if [[ "${CHANGE_TARGET}" =~ branch-2* ]]; then
 136     YETUS_ARGS+=("--hadoop-profile=${HADOOP_PROFILE}")
 137   fi
 138 fi
 139 if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
 140   YETUS_ARGS+=("--exclude-tests-url=${EXCLUDE_TESTS_URL}")
 141 fi
 142 # help keep the ASF boxes clean
 143 YETUS_ARGS+=("--sentinel")
 144 # use emoji vote so it is easier to find the broken line
 145 YETUS_ARGS+=("--github-use-emoji-vote")
 146
 147 echo "Launching yetus with command line:"
 148 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
 149
 150 /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"