dev-support/hbase-vote.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e -o pipefail
  20
  21 usage() {
  22   SCRIPT=$(basename "${BASH_SOURCE[@]}")
  23
  24   cat << __EOF
  25 hbase-vote. A script for standard vote which verifies the following items
  26 1. Checksum of sources and binaries
  27 2. Signature of sources and binaries
  28 3. Rat check
  29 4. Built from source
  30 5. Unit tests
  31
  32 Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>]
  33        ${SCRIPT} -h | --help
  34
  35   -h | --help                   Show this screen.
  36   -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
  37                                 e.g. https://dist.apache.org/repos/dist/dev/hbase/hbase-<version>RC0/
  38   -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
  39   -f | --keys-file-url '<url>'   the URL of the key file, default is
  40                                 http://www.apache.org/dist/hbase/KEYS
  41   -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
  42 __EOF
  43 }
  44
  45 while ((${#})); do
  46   case "${1}" in
  47     -h | --help )
  48       usage; exit 0                 ;;
  49     -s | --source  )
  50       SOURCE_URL="${2}"; shift 2    ;;
  51     -k | --key  )
  52       SIGNING_KEY="${2}"; shift 2   ;;
  53     -f | --keys-file-url )
  54       KEY_FILE_URL="${2}"; shift 2  ;;
  55     -o | --output-dir )
  56       OUTPUT_DIR="${2}"; shift 2    ;;
  57     *           )
  58       usage >&2; exit 1             ;;
  59   esac
  60 done
  61
  62 # Source url must be provided
  63 if [ -z "${SOURCE_URL}" ]; then
  64   usage;
  65   exit 1
  66 fi
  67
  68 cat << __EOF
  69 Although This tool helps verifying HBase RC build and unit tests,
  70 operator may still consider to verify the following manually
  71 1. Verify the compat-check-report.html
  72 2. Any on cluster Integration test or performance test, e.g. LTT load 100M or
  73    ITBLL 1B rows with serverKilling monkey
  74 3. Other concerns if any
  75 __EOF
  76
  77 [[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
  78 HBASE_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
  79 HBASE_VERSION=$(echo "${HBASE_RC_VERSION}" | sed -e 's/RC[0-9]//g' | sed -e 's/hbase-//g')
  80 JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
  81 OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
  82
  83 if [ ! -d "${OUTPUT_DIR}" ]; then
  84     echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
  85     exit 1
  86 fi
  87
  88 OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HBASE_RC_VERSION}"
  89
  90 # default value for verification targets, 0 = failed
  91 SIGNATURE_PASSED=0
  92 CHECKSUM_PASSED=0
  93 RAT_CHECK_PASSED=0
  94 BUILD_FROM_SOURCE_PASSED=0
  95 UNIT_TEST_PASSED=0
  96
  97 function download_and_import_keys() {
  98     KEY_FILE_URL="${KEY_FILE_URL:-https://www.apache.org/dist/hbase/KEYS}"
  99     echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
 100     # download the keys file into file KEYS
 101     wget -O KEYS "${KEY_FILE_URL}"
 102     gpg --import KEYS
 103     if [ -n "${SIGNING_KEY}" ]; then
 104         gpg --list-keys "${SIGNING_KEY}"
 105     fi
 106 }
 107
 108 function download_release_candidate () {
 109     # get all files from release candidate repo
 110     wget -r -np -nH --cut-dirs 4 "${SOURCE_URL}"
 111 }
 112
 113 function verify_signatures() {
 114     rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
 115     for file in *.tar.gz; do
 116         gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
 117     done
 118 }
 119
 120 function verify_checksums() {
 121     rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
 122     SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
 123     for file in *.tar.gz; do
 124         gpg --print-md SHA512 "${file}" > "${file}"."${SHA_EXT}".tmp
 125         diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
 126         rm -f "${file}"."${SHA_EXT}".tmp
 127     done
 128 }
 129
 130 function unzip_from_source() {
 131     tar -zxvf hbase-"${HBASE_VERSION}"-src.tar.gz
 132     cd hbase-"${HBASE_VERSION}"
 133 }
 134
 135 function rat_test() {
 136     rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
 137     mvn clean apache-rat:check 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
 138 }
 139
 140 function build_from_source() {
 141     rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
 142     mvn clean install -DskipTests 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
 143 }
 144
 145 function run_all_tests() {
 146     rm -f "${OUTPUT_PATH_PREFIX}"_run_all_tests
 147     mvn test -fae -P runAllTests -Dsurefire.rerunFailingTestsCount=3 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_run_all_tests && UNIT_TEST_PASSED=1
 148 }
 149
 150 function execute() {
 151    ${1} || print_when_exit
 152 }
 153
 154 function print_when_exit() {
 155   cat << __EOF
 156         * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
 157         * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
 158         * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
 159          - mvn clean apache-rat:check
 160         * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
 161          - mvn clean install -DskipTests
 162         * Unit tests pass (${JAVA_VERSION}): $( ((UNIT_TEST_PASSED)) && echo "ok" || echo "failed" )
 163          - mvn test -P runAllTests
 164 __EOF
 165   if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((UNIT_TEST_PASSED)) ; then
 166     exit 0
 167   fi
 168   exit 1
 169 }
 170
 171 download_and_import_keys
 172 download_release_candidate
 173 pushd "${HBASE_RC_VERSION}"
 174
 175 execute verify_signatures
 176 execute verify_checksums
 177 execute unzip_from_source
 178 execute rat_test
 179 execute build_from_source
 180 execute run_all_tests
 181
 182 popd
 183
 184 print_when_exit