dev-support/hbase-vote.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e -o pipefail
  20
  21 usage() {
  22   SCRIPT=$(basename "${BASH_SOURCE[@]}")
  23
  24   cat << __EOF
  25 hbase-vote. A script for standard vote which verifies the following items
  26 1. Checksum of sources and binaries
  27 2. Signature of sources and binaries
  28 3. Rat check
  29 4. Built from source
  30 5. Unit tests
  31
  32 Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>] [-P runSmallTests]
  33        ${SCRIPT} -h | --help
  34
  35   -h | --help                   Show this screen.
  36   -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
  37                                 e.g. https://dist.apache.org/repos/dist/dev/hbase/hbase-<version>RC0/
  38   -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
  39   -f | --keys-file-url '<url>'   the URL of the key file, default is
  40                                 http://www.apache.org/dist/hbase/KEYS
  41   -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
  42   -P |                          list of maven profiles to activate for test UT/IT, i.e. <-P runSmallTests> Defaults to runAllTests
  43 __EOF
  44 }
  45
  46 while ((${#})); do
  47   case "${1}" in
  48     -h | --help )
  49       usage; exit 0                 ;;
  50     -s | --source  )
  51       SOURCE_URL="${2}"; shift 2    ;;
  52     -k | --key  )
  53       SIGNING_KEY="${2}"; shift 2   ;;
  54     -f | --keys-file-url )
  55       KEY_FILE_URL="${2}"; shift 2  ;;
  56     -o | --output-dir )
  57       OUTPUT_DIR="${2}"; shift 2    ;;
  58     -P )
  59       MVN_ARGS="-P ${2}"; shift 2   ;;
  60     *           )
  61       usage >&2; exit 1             ;;
  62   esac
  63 done
  64
  65 # Source url must be provided
  66 if [ -z "${SOURCE_URL}" ]; then
  67   usage;
  68   exit 1
  69 fi
  70
  71 cat << __EOF
  72 Although This tool helps verifying HBase RC build and unit tests,
  73 operator may still consider to verify the following manually
  74 1. Verify the compat-check-report.html
  75 2. Any on cluster Integration test or performance test, e.g. LTT load 100M or
  76    ITBLL 1B rows with serverKilling monkey
  77 3. Other concerns if any
  78 __EOF
  79
  80 [[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
  81 HBASE_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
  82 HBASE_VERSION=$(echo "${HBASE_RC_VERSION}" | sed -e 's/RC[0-9]//g' | sed -e 's/hbase-//g')
  83 JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
  84 OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
  85
  86 if [ ! -d "${OUTPUT_DIR}" ]; then
  87     echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
  88     exit 1
  89 fi
  90
  91 # Maven profile must be provided
  92 if [ -z "${MVN_ARGS}" ]; then
  93     MVN_ARGS="-P runAllTests"
  94 fi
  95
  96 OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HBASE_RC_VERSION}"
  97
  98 # default value for verification targets, 0 = failed
  99 SIGNATURE_PASSED=0
 100 CHECKSUM_PASSED=0
 101 RAT_CHECK_PASSED=0
 102 BUILD_FROM_SOURCE_PASSED=0
 103 UNIT_TEST_PASSED=0
 104
 105 function download_and_import_keys() {
 106     KEY_FILE_URL="${KEY_FILE_URL:-https://www.apache.org/dist/hbase/KEYS}"
 107     echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
 108     # download the keys file into file KEYS
 109     wget -O KEYS "${KEY_FILE_URL}"
 110     gpg --import KEYS
 111     if [ -n "${SIGNING_KEY}" ]; then
 112         gpg --list-keys "${SIGNING_KEY}"
 113     fi
 114 }
 115
 116 function download_release_candidate () {
 117     # get all files from release candidate repo
 118     wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}"
 119 }
 120
 121 function verify_signatures() {
 122     rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
 123     for file in *.tar.gz; do
 124         gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
 125     done
 126 }
 127
 128 function verify_checksums() {
 129     rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
 130     SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
 131     for file in *.tar.gz; do
 132         gpg --print-md SHA512 "${file}" > "${file}"."${SHA_EXT}".tmp
 133         diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
 134         rm -f "${file}"."${SHA_EXT}".tmp
 135     done
 136 }
 137
 138 function unzip_from_source() {
 139     tar -zxvf hbase-"${HBASE_VERSION}"-src.tar.gz
 140     cd hbase-"${HBASE_VERSION}"
 141 }
 142
 143 function rat_test() {
 144     rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
 145     mvn clean apache-rat:check 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
 146 }
 147
 148 function build_from_source() {
 149     rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
 150     mvn clean install -DskipTests 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
 151 }
 152
 153 function run_tests() {
 154     rm -f "${OUTPUT_PATH_PREFIX}"_run_tests
 155     mvn package "${MVN_ARGS}" -Dsurefire.rerunFailingTestsCount=3 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_run_tests && UNIT_TEST_PASSED=1
 156 }
 157
 158 function execute() {
 159    ${1} || print_when_exit
 160 }
 161
 162 function print_when_exit() {
 163   cat << __EOF
 164         * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
 165         * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
 166         * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
 167          - mvn clean apache-rat:check
 168         * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
 169          - mvn clean install -DskipTests
 170         * Unit tests pass (${JAVA_VERSION}): $( ((UNIT_TEST_PASSED)) && echo "ok" || echo "failed" )
 171          - mvn package ${MVN_ARGS}
 172 __EOF
 173   if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((UNIT_TEST_PASSED)) ; then
 174     exit 0
 175   fi
 176   exit 1
 177 }
 178
 179 download_and_import_keys
 180 download_release_candidate
 181 pushd "${HBASE_RC_VERSION}"
 182
 183 execute verify_signatures
 184 execute verify_checksums
 185 execute unzip_from_source
 186 execute rat_test
 187 execute build_from_source
 188 execute run_tests
 189
 190 popd
 191
 192 print_when_exit