dev-support/jenkins_precommit_jira_yetus.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 if [[ "true" = "${DEBUG}" ]]; then
  20   set -x
  21   printenv
  22 fi
  23
  24 ##To set jenkins Environment Variables:
  25 export TOOLS_HOME=/home/jenkins/tools
  26 #export JAVA_HOME=${JAVA_HOME_HADOOP_MACHINES_HOME}
  27 export FINDBUGS_HOME=${TOOLS_HOME}/findbugs/latest
  28 export CLOVER_HOME=${TOOLS_HOME}/clover/latest
  29 #export MAVEN_HOME=${MAVEN_3_0_4_HOME}
  30 export MAVEN_HOME=/home/jenkins/tools/maven/apache-maven-3.0.5
  31
  32 #export PATH=$PATH:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:
  33 export PATH=$PATH:${MAVEN_HOME}/bin:
  34
  35 YETUS_RELEASE=0.11.1
  36 COMPONENT=${WORKSPACE}/component
  37 TEST_FRAMEWORK=${WORKSPACE}/test_framework
  38
  39 PATCHPROCESS=${WORKSPACE}/patchprocess
  40 if [[ -d ${PATCHPROCESS} ]]; then
  41   echo "[WARN] patch process already existed '${PATCHPROCESS}'"
  42   rm -rf "${PATCHPROCESS}"
  43 fi
  44 mkdir -p "${PATCHPROCESS}"
  45
  46
  47 ## Checking on H* machine nonsense
  48 echo "JAVA_HOME: ${JAVA_HOME}"
  49 ls -l "${JAVA_HOME}" || true
  50 echo "MAVEN_HOME: ${MAVEN_HOME}"
  51 echo "maven version:"
  52 mvn --offline --version  || true
  53 echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/"
  54 mkdir "${PATCHPROCESS}/machine"
  55 cat /proc/cpuinfo >"${PATCHPROCESS}/machine/cpuinfo" 2>&1 || true
  56 cat /proc/meminfo >"${PATCHPROCESS}/machine/meminfo" 2>&1 || true
  57 cat /proc/diskstats >"${PATCHPROCESS}/machine/diskstats" 2>&1 || true
  58 cat /sys/block/sda/stat >"${PATCHPROCESS}/machine/sys-block-sda-stat" 2>&1 || true
  59 df -h >"${PATCHPROCESS}/machine/df-h" 2>&1 || true
  60 ps -Awwf >"${PATCHPROCESS}/machine/ps-Awwf" 2>&1 || true
  61 ifconfig -a >"${PATCHPROCESS}/machine/ifconfig-a" 2>&1 || true
  62 lsblk -ta >"${PATCHPROCESS}/machine/lsblk-ta" 2>&1 || true
  63 lsblk -fa >"${PATCHPROCESS}/machine/lsblk-fa" 2>&1 || true
  64 cat /proc/loadavg >"${PATCHPROCESS}/loadavg" 2>&1 || true
  65 ulimit -a >"${PATCHPROCESS}/machine/ulimit-a" 2>&1 || true
  66
  67 ## /H*
  68
  69 ### Download Yetus
  70 if [[ "true" != "${USE_YETUS_PRERELEASE}" ]]; then
  71   if [ ! -d "${TEST_FRAMEWORK}/yetus-${YETUS_RELEASE}" ]; then
  72     mkdir -p "${TEST_FRAMEWORK}"
  73     cd "${TEST_FRAMEWORK}" || exit 1
  74     # clear out any cached 'use a prerelease' versions
  75     rm -rf apache-yetus-*
  76
  77     mkdir -p "${TEST_FRAMEWORK}/.gpg"
  78     chmod -R 700 "${TEST_FRAMEWORK}/.gpg"
  79
  80     curl -L --fail -o "${TEST_FRAMEWORK}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS
  81     gpg --homedir "${TEST_FRAMEWORK}/.gpg" --import "${TEST_FRAMEWORK}/KEYS_YETUS"
  82
  83     ## Release
  84     curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
  85     curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
  86     gpg --homedir "${TEST_FRAMEWORK}/.gpg" --verify "apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
  87     tar xzpf "apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
  88   fi
  89   TESTPATCHBIN=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/bin/test-patch
  90   TESTPATCHLIB=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/lib/precommit
  91 else
  92   prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
  93   if [ ! -d "${prerelease_dirs[0]}" ]; then
  94     mkdir -p "${TEST_FRAMEWORK}"
  95     cd "${TEST_FRAMEWORK}" || exit
  96     ## from github
  97     curl -L --fail "https://api.github.com/repos/${YETUS_PRERELEASE_GITHUB}/tarball/HEAD" > yetus.tar.gz
  98     tar xvpf yetus.tar.gz
  99     prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
 100   fi
 101   TESTPATCHBIN=${prerelease_dirs[0]}/precommit/test-patch.sh
 102   TESTPATCHLIB=${prerelease_dirs[0]}/precommit
 103 fi
 104
 105 if [[ "true" = "${DEBUG}" ]]; then
 106   # DEBUG print the test framework
 107   ls -l "${TESTPATCHBIN}"
 108   ls -la "${TESTPATCHLIB}/test-patch.d/"
 109   # DEBUG print the local customization
 110   if [ -d "${COMPONENT}/dev-support/test-patch.d" ]; then
 111     ls -la "${COMPONENT}/dev-support/test-patch.d/"
 112   fi
 113   YETUS_ARGS=(--debug "${YETUS_ARGS[@]}")
 114 fi
 115
 116
 117 if [ ! -x "${TESTPATCHBIN}" ] && [ -n "${TEST_FRAMEWORK}" ] && [ -d "${TEST_FRAMEWORK}" ]; then
 118   echo "Something is amiss with the test framework; removing it. please re-run."
 119   rm -rf "${TEST_FRAMEWORK}"
 120   exit 1
 121 fi
 122
 123 cd "${WORKSPACE}" || exit
 124
 125
 126 #
 127 # Yetus *always* builds with JAVA_HOME, so no need to list it.
 128 #
 129 # non-docker-mode JDK:
 130 #         --findbugs-home=/home/jenkins/tools/findbugs/latest \
 131
 132 # docker-mode:  (openjdk 7 added for free)
 133 #         --findbugs-home=/usr \
 134 #         --docker \
 135 #         --multijdkdirs="/usr/lib/jvm/java-8-openjdk-amd64" \
 136
 137 if [[ "true" = "${RUN_IN_DOCKER}" ]]; then
 138   YETUS_ARGS=(
 139     --docker \
 140     "--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64" \
 141     "--findbugs-home=/usr" \
 142     "${YETUS_ARGS[@]}" \
 143   )
 144   if [ -r "${COMPONENT}/dev-support/docker/Dockerfile" ]; then
 145     YETUS_ARGS=("--dockerfile=${COMPONENT}/dev-support/docker/Dockerfile" "${YETUS_ARGS[@]}")
 146   fi
 147 else
 148   YETUS_ARGS=("--findbugs-home=/home/jenkins/tools/findbugs/latest" "${YETUS_ARGS[@]}")
 149 fi
 150
 151 if [ -d "${COMPONENT}/dev-support/test-patch.d" ]; then
 152   YETUS_ARGS=("--user-plugins=${COMPONENT}/dev-support/test-patch.d" "${YETUS_ARGS[@]}")
 153 fi
 154
 155 # I don't trust Yetus compat enough yet, so in prerelease mode, skip our personality.
 156 # this should give us an incentive to update the Yetus exemplar for HBase periodically.
 157 if [ -r "${COMPONENT}/dev-support/hbase-personality.sh" ] && [[ "true" != "${USE_YETUS_PRERELEASE}" ]] ; then
 158   YETUS_ARGS=("--personality=${COMPONENT}/dev-support/hbase-personality.sh" "${YETUS_ARGS[@]}")
 159 fi
 160
 161 if [[ true == "${QUICK_HADOOPCHECK}" ]]; then
 162   YETUS_ARGS=("--quick-hadoopcheck" "${YETUS_ARGS[@]}")
 163 fi
 164
 165 if [[ true == "${SKIP_ERRORPRONE}" ]]; then
 166   YETUS_ARGS=("--skip-errorprone" "${YETUS_ARGS[@]}")
 167 fi
 168
 169 YETUS_ARGS=("--skip-dirs=dev-support" "${YETUS_ARGS[@]}")
 170
 171 /bin/bash "${TESTPATCHBIN}" \
 172         "${YETUS_ARGS[@]}" \
 173         --patch-dir="${PATCHPROCESS}" \
 174         --basedir="${COMPONENT}" \
 175         --mvn-custom-repos \
 176         --whitespace-eol-ignore-list=".*/generated/.*" \
 177         --whitespace-tabs-ignore-list=".*/generated/.*" \
 178         --jira-user=HBaseQA \
 179         --jira-password="${JIRA_PASSWORD}" \
 180         "HBASE-${ISSUE_NUM}"
 181
 182 find "${COMPONENT}" -name target -exec chmod -R u+w {} \;