Revert "Downloads page archive link should be https"
[hbase.git] / dev-support / hbase_nightly_source-artifact.sh
blobcd17a32cbff3b88e18ecf499773b14c9e332b1b0
1 #!/usr/bin/env bash
2 # Licensed to the Apache Software Foundation (ASF) under one
3 # or more contributor license agreements. See the NOTICE file
4 # distributed with this work for additional information
5 # regarding copyright ownership. The ASF licenses this file
6 # to you under the Apache License, Version 2.0 (the
7 # "License"); you may not use this file except in compliance
8 # with the License. You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing,
13 # software distributed under the License is distributed on an
14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 # KIND, either express or implied. See the License for the
16 # specific language governing permissions and limitations
17 # under the License.
19 set -e
20 function usage {
21 echo "Usage: ${0} [options] /path/to/component/checkout"
22 echo ""
23 echo " --intermediate-file-dir /path/to/use Path for writing listings and diffs. must exist."
24 echo " defaults to making a directory via mktemp."
25 echo " --unpack-temp-dir /path/to/use Path for unpacking tarball. default to"
26 echo " 'unpacked_src_tarball' in intermediate directory."
27 echo " --maven-m2-initial /path/to/use Path for maven artifacts while building in"
28 echo " component-dir."
29 echo " --maven-m2-src-build /path/to/use Path for maven artifacts while building from the"
30 echo " unpacked source tarball."
31 echo " --clean-source-checkout Destructively clean component checkout before"
32 echo " comparing to source tarball. N.B. will delete"
33 echo " anything in the checkout dir that isn't from"
34 echo " a git checkout, including ignored files."
35 exit 1
37 # if no args specified, show usage
38 if [ $# -lt 1 ]; then
39 usage
42 # Get arguments
43 declare component_dir
44 declare unpack_dir
45 declare m2_initial
46 declare m2_tarbuild
47 declare working_dir
48 declare source_clean
49 while [ $# -gt 0 ]
51 case "$1" in
52 --unpack-temp-dir) shift; unpack_dir=$1; shift;;
53 --maven-m2-initial) shift; m2_initial=$1; shift;;
54 --maven-m2-src-build) shift; m2_tarbuild=$1; shift;;
55 --intermediate-file-dir) shift; working_dir=$1; shift;;
56 --clean-source-checkout) shift; source_clean="true";;
57 --) shift; break;;
58 -*) usage ;;
59 *) break;; # terminate while loop
60 esac
61 done
63 # should still have where component checkout is.
64 if [ $# -lt 1 ]; then
65 usage
67 component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
69 if [ -z "${working_dir}" ]; then
70 if ! working_dir="$(mktemp -d -t hbase-srctarball-test)" ; then
71 echo "Failed to create temporary working directory. Please specify via --unpack-temp-dir"
72 exit 1
74 else
75 # absolutes please
76 working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
77 if [ ! -d "${working_dir}" ]; then
78 echo "passed working directory '${working_dir}' must already exist."
79 exit 1
83 echo "You'll find logs and temp files in ${working_dir}"
85 if [ -z "${unpack_dir}" ]; then
86 unpack_dir="${working_dir}/unpacked_src_tarball"
87 mkdir "${unpack_dir}"
88 else
89 # absolutes please
90 unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")"
91 if [ ! -d "${unpack_dir}" ]; then
92 echo "passed directory for unpacking the source tarball '${unpack_dir}' must already exist."
93 exit 1
95 rm -rf "${unpack_dir:?}/*"
98 if [ -z "${m2_initial}" ]; then
99 m2_initial="${working_dir}/.m2-initial"
100 mkdir "${m2_initial}"
101 else
102 # absolutes please
103 m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")"
104 if [ ! -d "${m2_initial}" ]; then
105 echo "passed directory for storing the initial build's maven repo '${m2_initial}' " \
106 "must already exist."
107 exit 1
111 if [ -z "${m2_tarbuild}" ]; then
112 m2_tarbuild="${working_dir}/.m2-tarbuild"
113 mkdir "${m2_tarbuild}"
114 else
115 # absolutes please
116 m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")"
117 if [ ! -d "${m2_tarbuild}" ]; then
118 echo "passed directory for storing the build from src tarball's maven repo '${m2_tarbuild}' " \
119 "must already exist."
120 exit 1
124 # This is meant to mimic what a release manager will do to create RCs.
125 # See http://hbase.apache.org/book.html#maven.release
127 echo "Maven details, in case our JDK doesn't match expectations:"
128 mvn --version --offline | tee "${working_dir}/maven_version"
130 echo "Do a clean building of the source artifact using code in ${component_dir}"
131 cd "${component_dir}"
132 if [ -n "${source_clean}" ]; then
133 echo "Clean..."
134 git clean -xdfff >"${working_dir}/component_git_clean.log" 2>&1
136 echo "Follow the ref guide section on making a RC: Step 6 Build the source tarball"
137 git archive --format=tar.gz --output="${working_dir}/hbase-src.tar.gz" \
138 --prefix="hbase-SOMEVERSION/" HEAD \
139 >"${working_dir}/component_build_src_tarball.log" 2>&1
141 cd "${unpack_dir}"
142 echo "Unpack the source tarball"
143 tar --strip-components=1 -xzf "${working_dir}/hbase-src.tar.gz" \
144 >"${working_dir}/srctarball_unpack.log" 2>&1
146 cd "${component_dir}"
147 echo "Diff against source tree"
148 diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true
150 cd "${working_dir}"
151 # expectation check largely based on HBASE-14952
152 echo "Checking against things we don't expect to include in the source tarball (git related, etc.)"
153 # Add in lines to show differences between the source tarball and this branch, in the same format diff would give.
154 # e.g. prior to HBASE-19152 we'd have the following lines (ignoring the bash comment marker):
155 #Only in .: .gitattributes
156 #Only in .: .gitignore
157 #Only in .: hbase-native-client
158 cat >known_excluded <<END
159 Only in .: .git
161 if ! diff known_excluded diff_output >"${working_dir}/unexpected.diff" ; then
162 echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
163 echo "One potential source of differences is if you have an unclean working directory; you should expect to see"
164 echo "such extraneous files below."
165 echo ""
166 echo "The expected differences are on the < side and the current differences are on the > side."
167 echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
168 diff known_excluded diff_output
169 else
170 echo "Everything looks as expected."
173 cd "${unpack_dir}"
174 echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
175 if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
176 assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
177 for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
178 if [ -f "${artifact}" ]; then
179 # TODO check the layout of the binary artifact we just made.
180 echo "Building a binary tarball from the source tarball succeeded."
181 exit 0
183 done
185 echo "Building a binary tarball from the source tarball failed. see srctarball_install.log for details."
186 # Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
187 # rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
188 find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
189 exit 1