dev-support/hbase-vote.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e -o pipefail
  20
  21 usage() {
  22   SCRIPT=$(basename "${BASH_SOURCE[@]}")
  23
  24   cat << __EOF
  25 hbase-vote. A script for standard vote which verifies the following items
  26 1. Checksum of sources and binaries
  27 2. Signature of sources and binaries
  28 3. Rat check
  29 4. Built from source
  30 5. Unit tests
  31
  32 Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>] [-P runSmallTests] [-D property[=value]]
  33        ${SCRIPT} -h | --help
  34
  35   -h | --help                   Show this screen.
  36   -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
  37                                 e.g. https://dist.apache.org/repos/dist/dev/hbase/hbase-<version>RC0/
  38   -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
  39   -f | --keys-file-url '<url>'   the URL of the key file, default is
  40                                 https://downloads.apache.org/hbase/KEYS
  41   -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
  42   -P |                          list of maven profiles to activate for test UT/IT, i.e. <-P runSmallTests> Defaults to runAllTests
  43   -D |                          list of maven properties to set for the mvn invocations, i.e. <-D hadoop.profile=3.0 -D skipTests> Defaults to unset
  44 __EOF
  45 }
  46
  47 MVN_PROFILES=()
  48 MVN_PROPERTIES=()
  49
  50 while ((${#})); do
  51   case "${1}" in
  52     -h | --help )
  53       usage; exit 0 ;;
  54     -s | --source )
  55       SOURCE_URL="${2}"; shift 2 ;;
  56     -k | --key )
  57       SIGNING_KEY="${2}"; shift 2 ;;
  58     -f | --keys-file-url )
  59       KEY_FILE_URL="${2}"; shift 2 ;;
  60     -o | --output-dir )
  61       OUTPUT_DIR="${2}"; shift 2 ;;
  62     -P )
  63       MVN_PROFILES+=("-P ${2}"); shift 2 ;;
  64     -D )
  65       MVN_PROPERTIES+=("-D ${2}"); shift 2 ;;
  66     * )
  67       usage >&2; exit 1             ;;
  68   esac
  69 done
  70
  71 # Source url must be provided
  72 if [ -z "${SOURCE_URL}" ]; then
  73   usage;
  74   exit 1
  75 fi
  76
  77 cat << __EOF
  78 Although This tool helps verifying HBase RC build and unit tests,
  79 operator may still consider to verify the following manually
  80 1. Verify the compat-check-report.html
  81 2. Any on cluster Integration test or performance test, e.g. LTT load 100M or
  82    ITBLL 1B rows with serverKilling monkey
  83 3. Other concerns if any
  84 __EOF
  85
  86 [[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
  87 HBASE_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
  88 HBASE_VERSION=$(echo "${HBASE_RC_VERSION}" | sed -e 's/RC[0-9]//g' | sed -e 's/hbase-//g')
  89 JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
  90 OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
  91
  92 if [ ! -d "${OUTPUT_DIR}" ]; then
  93     echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
  94     exit 1
  95 fi
  96
  97 # Maven profile must be provided
  98 if [ ${#MVN_PROFILES[@]} -eq 0 ]; then
  99     MVN_PROFILES=("-P runAllTests")
 100 fi
 101
 102 OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HBASE_RC_VERSION}"
 103
 104 # default value for verification targets, 0 = failed
 105 SIGNATURE_PASSED=0
 106 CHECKSUM_PASSED=0
 107 RAT_CHECK_PASSED=0
 108 BUILD_FROM_SOURCE_PASSED=0
 109 UNIT_TEST_PASSED=0
 110
 111 function download_and_import_keys() {
 112     KEY_FILE_URL="${KEY_FILE_URL:-https://downloads.apache.org/hbase/KEYS}"
 113     echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
 114     # download the keys file into file KEYS
 115     wget -O KEYS "${KEY_FILE_URL}"
 116     gpg --import KEYS
 117     if [ -n "${SIGNING_KEY}" ]; then
 118         gpg --list-keys "${SIGNING_KEY}"
 119     fi
 120 }
 121
 122 function download_release_candidate () {
 123     # get all files from release candidate repo
 124     wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}"
 125 }
 126
 127 function verify_signatures() {
 128     rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
 129     for file in *.tar.gz; do
 130         gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
 131     done
 132 }
 133
 134 function verify_checksums() {
 135     rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
 136     SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
 137     for file in *.tar.gz; do
 138         gpg --print-md SHA512 "${file}" > "${file}"."${SHA_EXT}".tmp
 139         diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
 140         rm -f "${file}"."${SHA_EXT}".tmp
 141     done
 142 }
 143
 144 function unzip_from_source() {
 145     tar -zxvf hbase-"${HBASE_VERSION}"-src.tar.gz
 146     cd hbase-"${HBASE_VERSION}"
 147 }
 148
 149 function rat_test() {
 150     rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
 151     mvn clean apache-rat:check "${MVN_PROPERTIES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
 152 }
 153
 154 function build_from_source() {
 155     rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
 156     # Hardcode skipTests for faster build. Testing is covered later.
 157     mvn clean install "${MVN_PROPERTIES[@]}" -DskipTests 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
 158 }
 159
 160 function run_tests() {
 161     rm -f "${OUTPUT_PATH_PREFIX}"_run_tests
 162     mvn package "${MVN_PROFILES[@]}" "${MVN_PROPERTIES[@]}" -Dsurefire.rerunFailingTestsCount=3 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_run_tests && UNIT_TEST_PASSED=1
 163 }
 164
 165 function execute() {
 166    ${1} || print_when_exit
 167 }
 168
 169 function print_when_exit() {
 170   cat << __EOF
 171         * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
 172         * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
 173         * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
 174          - mvn clean apache-rat:check ${MVN_PROPERTIES[@]}
 175         * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
 176          - mvn clean install ${MVN_PROPERTIES[@]} -DskipTests
 177         * Unit tests pass (${JAVA_VERSION}): $( ((UNIT_TEST_PASSED)) && echo "ok" || echo "failed" )
 178          - mvn package ${MVN_PROFILES[@]} ${MVN_PROPERTIES[@]} -Dsurefire.rerunFailingTestsCount=3
 179 __EOF
 180   if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((UNIT_TEST_PASSED)) ; then
 181     exit 0
 182   fi
 183   exit 1
 184 }
 185
 186 download_and_import_keys
 187 download_release_candidate
 188 pushd "${HBASE_RC_VERSION}"
 189
 190 execute verify_signatures
 191 execute verify_checksums
 192 execute unzip_from_source
 193 execute rat_test
 194 execute build_from_source
 195 execute run_tests
 196
 197 popd
 198
 199 print_when_exit