dev-support/hbase_nightly_source-artifact.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e
  20 function usage {
  21   echo "Usage: ${0} [options] /path/to/component/checkout"
  22   echo ""
  23   echo "    --intermediate-file-dir /path/to/use  Path for writing listings and diffs. must exist."
  24   echo "                                          defaults to making a directory via mktemp."
  25   echo "    --unpack-temp-dir /path/to/use        Path for unpacking tarball. default to"
  26   echo "                                          'unpacked_src_tarball' in intermediate directory."
  27   echo "    --maven-m2-initial /path/to/use       Path for maven artifacts while building in"
  28   echo "                                          component-dir."
  29   echo "    --maven-m2-src-build /path/to/use     Path for maven artifacts while building from the"
  30   echo "                                          unpacked source tarball."
  31   echo "    --clean-source-checkout               Destructively clean component checkout before"
  32   echo "                                          comparing to source tarball. N.B. will delete"
  33   echo "                                          anything in the checkout dir that isn't from"
  34   echo "                                          a git checkout, including ignored files."
  35   exit 1
  36 }
  37 # if no args specified, show usage
  38 if [ $# -lt 1 ]; then
  39   usage
  40 fi
  41
  42 # Get arguments
  43 declare component_dir
  44 declare unpack_dir
  45 declare m2_initial
  46 declare m2_tarbuild
  47 declare working_dir
  48 declare source_clean
  49 while [ $# -gt 0 ]
  50 do
  51   case "$1" in
  52     --unpack-temp-dir) shift; unpack_dir=$1; shift;;
  53     --maven-m2-initial) shift; m2_initial=$1; shift;;
  54     --maven-m2-src-build) shift; m2_tarbuild=$1; shift;;
  55     --intermediate-file-dir) shift; working_dir=$1; shift;;
  56     --clean-source-checkout) shift; source_clean="true";;
  57     --) shift; break;;
  58     -*) usage ;;
  59     *)  break;;  # terminate while loop
  60   esac
  61 done
  62
  63 # should still have where component checkout is.
  64 if [ $# -lt 1 ]; then
  65   usage
  66 fi
  67 component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
  68
  69 if [ -z "${working_dir}" ]; then
  70   if ! working_dir="$(mktemp -d -t hbase-srctarball-test)" ; then
  71     echo "Failed to create temporary working directory. Please specify via --unpack-temp-dir"
  72     exit 1
  73   fi
  74 else
  75   # absolutes please
  76   working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
  77   if [ ! -d "${working_dir}" ]; then
  78     echo "passed working directory '${working_dir}' must already exist."
  79     exit 1
  80   fi
  81 fi
  82
  83 echo "You'll find logs and temp files in ${working_dir}"
  84
  85 if [ -z "${unpack_dir}" ]; then
  86   unpack_dir="${working_dir}/unpacked_src_tarball"
  87   mkdir "${unpack_dir}"
  88 else
  89   # absolutes please
  90   unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")"
  91   if [ ! -d "${unpack_dir}" ]; then
  92     echo "passed directory for unpacking the source tarball '${unpack_dir}' must already exist."
  93     exit 1
  94   fi
  95   rm -rf "${unpack_dir:?}/*"
  96 fi
  97
  98 if [ -z "${m2_initial}" ]; then
  99   m2_initial="${working_dir}/.m2-initial"
 100   mkdir "${m2_initial}"
 101 else
 102   # absolutes please
 103   m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")"
 104   if [ ! -d "${m2_initial}" ]; then
 105     echo "passed directory for storing the initial build's maven repo  '${m2_initial}' " \
 106         "must already exist."
 107     exit 1
 108   fi
 109 fi
 110
 111 if [ -z "${m2_tarbuild}" ]; then
 112   m2_tarbuild="${working_dir}/.m2-tarbuild"
 113   mkdir "${m2_tarbuild}"
 114 else
 115   # absolutes please
 116   m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")"
 117   if [ ! -d "${m2_tarbuild}" ]; then
 118     echo "passed directory for storing the build from src tarball's maven repo  '${m2_tarbuild}' " \
 119         "must already exist."
 120     exit 1
 121   fi
 122 fi
 123
 124 # This is meant to mimic what a release manager will do to create RCs.
 125 # See http://hbase.apache.org/book.html#maven.release
 126
 127 echo "Maven details, in case our JDK doesn't match expectations:"
 128 mvn --version --offline | tee "${working_dir}/maven_version"
 129
 130 echo "Do a clean building of the source artifact using code in ${component_dir}"
 131 cd "${component_dir}"
 132 if [ -n "${source_clean}" ]; then
 133   echo "Clean..."
 134   git clean -xdfff >"${working_dir}/component_git_clean.log" 2>&1
 135 fi
 136 echo "Follow the ref guide section on making a RC: Step 6 Build the source tarball"
 137 git archive --format=tar.gz --output="${working_dir}/hbase-src.tar.gz" \
 138     --prefix="hbase-SOMEVERSION/" HEAD \
 139     >"${working_dir}/component_build_src_tarball.log" 2>&1
 140
 141 cd "${unpack_dir}"
 142 echo "Unpack the source tarball"
 143 tar --strip-components=1 -xzf "${working_dir}/hbase-src.tar.gz" \
 144     >"${working_dir}/srctarball_unpack.log" 2>&1
 145
 146 cd "${component_dir}"
 147 echo "Diff against source tree"
 148 diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true
 149
 150 cd "${working_dir}"
 151 # expectation check largely based on HBASE-14952
 152 echo "Checking against things we don't expect to include in the source tarball (git related, etc.)"
 153 # Add in lines to show differences between the source tarball and this branch, in the same format diff would give.
 154 # e.g. prior to HBASE-19152 we'd have the following lines (ignoring the bash comment marker):
 155 #Only in .: .gitattributes
 156 #Only in .: .gitignore
 157 #Only in .: hbase-native-client
 158 cat >known_excluded <<END
 159 Only in .: .git
 160 END
 161 if ! diff known_excluded diff_output >"${working_dir}/unexpected.diff" ; then
 162   echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
 163   echo "One potential source of differences is if you have an unclean working directory; you should expect to see"
 164   echo "such extraneous files below."
 165   echo ""
 166   echo "The expected differences are on the < side and the current differences are on the > side."
 167   echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
 168   diff known_excluded diff_output
 169 else
 170   echo "Everything looks as expected."
 171 fi
 172
 173 cd "${unpack_dir}"
 174 echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
 175 if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
 176     assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
 177   for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
 178     if [ -f "${artifact}" ]; then
 179       # TODO check the layout of the binary artifact we just made.
 180       echo "Building a binary tarball from the source tarball succeeded."
 181       exit 0
 182     fi
 183   done
 184 fi
 185 echo "Building a binary tarball from the source tarball failed. see srctarball_install.log for details."
 186 # Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
 187 # rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
 188 find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
 189 exit 1