dev-support/hbase-vote.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e -o pipefail
  20
  21 usage() {
  22   SCRIPT=$(basename "${BASH_SOURCE[@]}")
  23
  24   cat << __EOF
  25 hbase-vote. A script for standard vote which verifies the following items
  26 1. Checksum of sources and binaries
  27 2. Signature of sources and binaries
  28 3. Rat check
  29 4. Built from source
  30 5. Unit tests
  31
  32 Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>] [-P runSmallTests] [-D property[=value]]
  33        ${SCRIPT} -h | --help
  34
  35   -h | --help                   Show this screen.
  36   -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
  37                                 e.g. https://dist.apache.org/repos/dist/dev/hbase/hbase-<version>RC0/
  38   -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
  39   -f | --keys-file-url '<url>'   the URL of the key file, default is
  40                                 https://downloads.apache.org/hbase/KEYS
  41   -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
  42   -P |                          list of maven profiles to activate for test UT/IT, i.e. <-P runSmallTests> Defaults to runAllTests
  43   -D |                          list of maven properties to set for the mvn invocations, i.e. <-D hadoop.profile=3.0> Defaults to unset
  44 __EOF
  45 }
  46
  47 while ((${#})); do
  48   case "${1}" in
  49     -h | --help )
  50       usage; exit 0 ;;
  51     -s | --source )
  52       SOURCE_URL="${2}"; shift 2 ;;
  53     -k | --key )
  54       SIGNING_KEY="${2}"; shift 2 ;;
  55     -f | --keys-file-url )
  56       KEY_FILE_URL="${2}"; shift 2 ;;
  57     -o | --output-dir )
  58       OUTPUT_DIR="${2}"; shift 2 ;;
  59     -P )
  60       MVN_PROFILES="-P ${2}"; shift 2 ;;
  61     -D )
  62       MVN_PROPERTIES="-D ${2}"; shift 2 ;;
  63     * )
  64       usage >&2; exit 1             ;;
  65   esac
  66 done
  67
  68 # Source url must be provided
  69 if [ -z "${SOURCE_URL}" ]; then
  70   usage;
  71   exit 1
  72 fi
  73
  74 cat << __EOF
  75 Although This tool helps verifying HBase RC build and unit tests,
  76 operator may still consider to verify the following manually
  77 1. Verify the compat-check-report.html
  78 2. Any on cluster Integration test or performance test, e.g. LTT load 100M or
  79    ITBLL 1B rows with serverKilling monkey
  80 3. Other concerns if any
  81 __EOF
  82
  83 [[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
  84 HBASE_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
  85 HBASE_VERSION=$(echo "${HBASE_RC_VERSION}" | sed -e 's/RC[0-9]//g' | sed -e 's/hbase-//g')
  86 JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
  87 OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
  88
  89 if [ ! -d "${OUTPUT_DIR}" ]; then
  90     echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
  91     exit 1
  92 fi
  93
  94 # Maven profile must be provided
  95 if [ -z "${MVN_PROFILES}" ]; then
  96     MVN_PROFILES="-P runAllTests"
  97 fi
  98
  99 OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HBASE_RC_VERSION}"
 100
 101 # default value for verification targets, 0 = failed
 102 SIGNATURE_PASSED=0
 103 CHECKSUM_PASSED=0
 104 RAT_CHECK_PASSED=0
 105 BUILD_FROM_SOURCE_PASSED=0
 106 UNIT_TEST_PASSED=0
 107
 108 function download_and_import_keys() {
 109     KEY_FILE_URL="${KEY_FILE_URL:-https://downloads.apache.org/hbase/KEYS}"
 110     echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
 111     # download the keys file into file KEYS
 112     wget -O KEYS "${KEY_FILE_URL}"
 113     gpg --import KEYS
 114     if [ -n "${SIGNING_KEY}" ]; then
 115         gpg --list-keys "${SIGNING_KEY}"
 116     fi
 117 }
 118
 119 function download_release_candidate () {
 120     # get all files from release candidate repo
 121     wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}"
 122 }
 123
 124 function verify_signatures() {
 125     rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
 126     for file in *.tar.gz; do
 127         gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
 128     done
 129 }
 130
 131 function verify_checksums() {
 132     rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
 133     SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
 134     for file in *.tar.gz; do
 135         gpg --print-md SHA512 "${file}" > "${file}"."${SHA_EXT}".tmp
 136         diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
 137         rm -f "${file}"."${SHA_EXT}".tmp
 138     done
 139 }
 140
 141 function unzip_from_source() {
 142     tar -zxvf hbase-"${HBASE_VERSION}"-src.tar.gz
 143     cd hbase-"${HBASE_VERSION}"
 144 }
 145
 146 function rat_test() {
 147     rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
 148     mvn clean apache-rat:check "${MVN_PROPERTIES}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
 149 }
 150
 151 function build_from_source() {
 152     rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
 153     mvn clean install "${MVN_PROPERTIES}" -DskipTests 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
 154 }
 155
 156 function run_tests() {
 157     rm -f "${OUTPUT_PATH_PREFIX}"_run_tests
 158     mvn package "${MVN_PROFILES}" "${MVN_PROPERTIES}" -Dsurefire.rerunFailingTestsCount=3 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_run_tests && UNIT_TEST_PASSED=1
 159 }
 160
 161 function execute() {
 162    ${1} || print_when_exit
 163 }
 164
 165 function print_when_exit() {
 166   cat << __EOF
 167         * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
 168         * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
 169         * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
 170          - mvn clean apache-rat:check "${MVN_PROPERTIES}"
 171         * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
 172          - mvn clean install -DskipTests "${MVN_PROPERTIES}"
 173         * Unit tests pass (${JAVA_VERSION}): $( ((UNIT_TEST_PASSED)) && echo "ok" || echo "failed" )
 174          - mvn package ${MVN_PROFILES} "${MVN_PROPERTIES}" -Dsurefire.rerunFailingTestsCount=3
 175 __EOF
 176   if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((UNIT_TEST_PASSED)) ; then
 177     exit 0
 178   fi
 179   exit 1
 180 }
 181
 182 download_and_import_keys
 183 download_release_candidate
 184 pushd "${HBASE_RC_VERSION}"
 185
 186 execute verify_signatures
 187 execute verify_checksums
 188 execute unzip_from_source
 189 execute rat_test
 190 execute build_from_source
 191 execute run_tests
 192
 193 popd
 194
 195 print_when_exit