3 # Licensed to the Apache Software Foundation (ASF) under one or more
4 # contributor license agreements. See the NOTICE file distributed with
5 # this work for additional information regarding copyright ownership.
6 # The ASF licenses this file to You under the Apache License, Version 2.0
7 # (the "License"); you may not use this file except in compliance with
8 # the License. You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
20 # Creates a HBase release candidate. The script will update versions, tag the branch,
21 # build HBase binary packages and documentation, and upload maven artifacts to a staging
22 # repository. There is also a dry run mode where only local builds are performed, and
23 # nothing is uploaded to the ASF repos.
25 # Run with "-h" for options. For example, running below will do all
26 # steps above using the 'rm' dir under Downloads as workspace:
28 # $ ./do-release-docker.sh -d ~/Downloads/rm
30 # The scripts in this directory came originally from spark [1]. They were then
31 # modified to suite the hbase context. These scripts supercedes the old
32 # ../make_rc.sh script for making release candidates because what is here is more
33 # comprehensive doing more steps of the RM process as well as running in a
34 # container so the RM build environment can be a constant.
38 # * Sets version to the release version
39 # * Sets version to next SNAPSHOT version.
40 # * Builds, signs, and hashes all artifacts.
41 # * Pushes release tgzs to the dev dir in a apache dist.
42 # * Pushes to repository.apache.org staging.
44 # The entry point is here, in the do-release-docker.sh script.
46 # 1. https://github.com/apache/spark/tree/master/dev/create-release
50 # Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools
51 export PROJECT
="${PROJECT:-hbase}"
53 SELF
="$(cd "$
(dirname "${BASH_SOURCE[0]}")" && pwd)"
54 # shellcheck source=SCRIPTDIR/release-util.sh
55 .
"$SELF/release-util.sh"
60 NAME
="$(basename "${BASH_SOURCE[0]}")"
62 Usage: $NAME [OPTIONS]
63 Runs release scripts inside a docker image.
65 -d [path] Required. Working directory. Output will be written to "output" in here.
66 -f "force" -- actually publish this release. Unless you specify '-f', it will
67 default to dry run mode, which checks and does local builds, but does not
69 -t [tag] Tag for the hbase-rm docker image to use for building (default: "latest").
70 -j [path] Path to local JDK installation to use building. By default the script will
71 use openjdk8 installed in the docker image.
72 -p [project] Project to build: e.g. 'hbase' or 'hbase-thirdparty'; defaults to PROJECT env var
73 -r [repo] Git repo to use for remote git operations. defaults to ASF gitbox for project.
74 -s [step] Runs a single step of the process; valid steps: tag|publish-dist|publish-release.
75 If none specified, runs tag, then publish-dist, and then publish-release.
76 'publish-snapshot' is also an allowed, less used, option.
77 -x Debug. Does less clean up (env file, gpg forwarding on mac)
87 while getopts "d:fhj:p:r:s:t:x" opt
; do
89 d
) WORKDIR
="$OPTARG" ;;
91 t
) IMGTAG
="$OPTARG" ;;
93 p
) PROJECT
="$OPTARG" ;;
94 r
) GIT_REPO
="$OPTARG" ;;
95 s
) RELEASE_STEP
="$OPTARG" ;;
98 ?
) error
"Invalid option. Run with -h for help." ;;
102 if (( $# > 0 )); then
103 error
"Arguments can only be provided with option flags, invalid args: $*"
107 if [ -z "$WORKDIR" ] ||
[ ! -d "$WORKDIR" ]; then
108 error
"Work directory (-d) must be defined and exist. Run with -h for help."
111 if [ -d "$WORKDIR/output" ]; then
112 read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
113 if [ "$ANSWER" != "y" ]; then
118 if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ] || \
119 [ -f "${WORKDIR}/gpg-proxy.cid" ] || \
120 [ -f "${WORKDIR}/release.cid" ]; then
121 read -r -p "container/pid files from prior run exists. Overwrite and continue? [y/n] " ANSWER
122 if [ "$ANSWER" != "y" ]; then
128 rm -rf "$WORKDIR/output"
129 rm -rf "${WORKDIR}/gpg-proxy.ssh.pid" "${WORKDIR}/gpg-proxy.cid" "${WORKDIR}/release.cid"
130 mkdir
"$WORKDIR/output"
132 banner "Gathering release details."
133 HOST_OS
="$(get_host_os)"
138 # Place all RM scripts and necessary data in a local directory that must be defined in the command
139 # line. This directory is mounted into the image. Its WORKDIR, the arg passed with -d.
140 for f
in "$SELF"/*; do
146 # We need to import that public key in the container in order to use the private key via the agent.
147 GPG_KEY_FILE
="$WORKDIR/gpg.key.public"
148 log
"Exporting public key for ${GPG_KEY}"
149 fcreate_secure
"$GPG_KEY_FILE"
150 $GPG "${GPG_ARGS[@]}" --export "${GPG_KEY}" > "${GPG_KEY_FILE}"
154 banner "Release Cleanup"
156 log
"skipping due to debug run"
159 log
"details in cleanup.log"
160 if [ -f "${ENVFILE}" ]; then
163 rm -f "$GPG_KEY_FILE"
164 if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ]; then
165 id
=$
(cat "${WORKDIR}/gpg-proxy.ssh.pid")
166 echo "Stopping ssh tunnel for gpg-agent at PID ${id}" |
tee -a cleanup.log
167 kill -9 "${id}" >>cleanup.log
2>&1 || true
168 rm -f "${WORKDIR}/gpg-proxy.ssh.pid" >>cleanup.log
2>&1
170 if [ -f "${WORKDIR}/gpg-proxy.cid" ]; then
171 id
=$
(cat "${WORKDIR}/gpg-proxy.cid")
172 echo "Stopping gpg-proxy container with ID ${id}" |
tee -a cleanup.log
173 docker
kill "${id}" >>cleanup.log
2>&1 || true
174 rm -f "${WORKDIR}/gpg-proxy.cid" >>cleanup.log
2>&1
175 # TODO we should remove the gpgagent volume?
177 if [ -f "${WORKDIR}/release.cid" ]; then
178 id
=$
(cat "${WORKDIR}/release.cid")
179 echo "Stopping release container with ID ${id}" |
tee -a cleanup.log
180 docker
kill "${id}" >>cleanup.log
2>&1 || true
181 rm -f "${WORKDIR}/release.cid" >>cleanup.log
2>&1
187 log
"Host OS: ${HOST_OS}"
188 if [ "${HOST_OS}" == "DARWIN" ]; then
189 run_silent
"Building gpg-agent-proxy image with tag ${IMGTAG}..." "docker-proxy-build.log" \
190 docker build
--build-arg "UID=${UID}" --build-arg "RM_USER=${USER}" \
191 --tag "org.apache.hbase/gpg-agent-proxy:${IMGTAG}" "${SELF}/mac-sshd-gpg-agent"
194 run_silent
"Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
195 docker build
--tag "org.apache.hbase/hbase-rm:$IMGTAG" --build-arg "UID=$UID" \
196 --build-arg "RM_USER=${USER}" "$SELF/hbase-rm"
198 banner "Final prep for container launch."
199 log
"Writing out environment for container."
200 # Write the release information to a file with environment variables to be used when running the
202 ENVFILE
="$WORKDIR/env.list"
203 fcreate_secure
"$ENVFILE"
205 cat > "$ENVFILE" <<EOF
210 GIT_BRANCH=$GIT_BRANCH
211 NEXT_VERSION=$NEXT_VERSION
212 RELEASE_VERSION=$RELEASE_VERSION
213 RELEASE_TAG=$RELEASE_TAG
215 ASF_USERNAME=$ASF_USERNAME
219 ASF_PASSWORD=$ASF_PASSWORD
220 RELEASE_STEP=$RELEASE_STEP
221 API_DIFF_TAG=$API_DIFF_TAG
226 if [ -n "$JAVA" ]; then
227 echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
228 JAVA_MOUNT
=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly")
231 #TODO some debug output would be good here
233 if [ -n "${GIT_REPO}" ]; then
234 case "${GIT_REPO}" in
235 # skip the easy to identify remote protocols
236 ssh://*|git
://*|http
://*|https
://*|
ftp://*|ftps
://*) ;;
239 GIT_REPO_MOUNT
=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo,consistency=delegated")
240 echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
241 GIT_REPO
="/opt/hbase-repo"
243 # on the host but normally git wouldn't use the local optimization
245 log
"Converted file:// git repo to a local path, which changes git to assume --local."
246 GIT_REPO_MOUNT
=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo,consistency=delegated")
247 echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
248 GIT_REPO
="/opt/hbase-repo"
250 # have to decide if it's a local path or the "scp-ish" remote
252 declare colon_remove_prefix
;
253 declare slash_remove_prefix
;
255 colon_remove_prefix
="${GIT_REPO#*:}"
256 slash_remove_prefix
="${GIT_REPO#*/}"
257 if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
258 # if there was no colon at all, we assume this must be a local path
259 local_path
="no colon at all"
260 elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
261 # if there was a colon and there is no slash, then we assume it must be scp-style host
262 # and a relative path
264 if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
265 # Given the substrings made by removing everything up to the first colon and slash
266 # we can determine which comes first based on the longer substring length.
267 # if the slash is first, then we assume the colon is part of a path name and if the colon
268 # is first then it is the seperator between a scp-style host name and the path.
269 local_path
="slash happened before a colon"
272 if [ -n "${local_path}" ]; then
273 # convert to an absolute path
274 GIT_REPO
="$(cd "$
(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
275 GIT_REPO_MOUNT=(--mount "type=bind,src
=${GIT_REPO},dst
=/opt
/hbase-repo
,consistency
=delegated
")
276 echo "HOST_GIT_REPO
=${GIT_REPO}" >> "${ENVFILE}"
277 GIT_REPO="/opt
/hbase-repo
"
281 echo "GIT_REPO
=${GIT_REPO}" >> "${ENVFILE}"
285 if [ "${HOST_OS}" == "DARWIN
" ]; then
286 GPG_PROXY_MOUNT=(--mount "type=volume
,src
=gpgagent
,dst
=/home
/${USER}/.gnupg
/")
287 log "Setting up GPG agent proxy container needed on OS X.
"
288 log " we should clean this up
for you. If that fails the container ID is below and
in " \
290 #TODO the key pair used should be configurable
291 docker run --rm -p 62222:22 \
292 --detach --cidfile "${WORKDIR}/gpg-proxy.cid
" \
294 "type=bind,src
=${HOME}/.ssh
/id_rsa.pub
,dst
=/home
/${USER}/.ssh
/authorized_keys
,readonly" \
295 "${GPG_PROXY_MOUNT[@]}" \
296 "org.apache.hbase
/gpg-agent-proxy
:${IMGTAG}"
297 # gotta trust the container host
298 ssh-keyscan -p 62222 localhost 2>/dev/null | sort > "${WORKDIR}/gpg-agent-proxy.ssh-keyscan
"
299 sort "${HOME}/.ssh
/known_hosts
" | comm -1 -3 - "${WORKDIR}/gpg-agent-proxy.ssh-keyscan
" \
300 > "${WORKDIR}/gpg-agent-proxy.known_hosts
"
301 if [ -s "${WORKDIR}/gpg-agent-proxy.known_hosts
" ]; then
302 log "Your
ssh known_hosts does not include the entries
for the gpg-agent proxy container.
"
303 log "The following entry
(ies
) are missing
:"
304 sed -e 's/^/ /' "${WORKDIR}/gpg-agent-proxy.known_hosts
"
305 read -r -p "Okay to add these entries to
${HOME}/.ssh
/known_hosts?
[y
/n
] " ANSWER
306 if [ "$ANSWER" != "y
" ]; then
309 cat "${WORKDIR}/gpg-agent-proxy.known_hosts
" >> "${HOME}/.ssh
/known_hosts
"
311 log "Launching
ssh reverse tunnel from the container to gpg agent.
"
312 log " we should clean this up
for you. If that fails the PID is
in gpg-proxy.
ssh.pid
"
313 ssh -p 62222 -R "/home
/${USER}/.gnupg
/S.gpg-agent
:$
(gpgconf
--list-dir agent-extra-socket
)" \
314 -i "${HOME}/.ssh
/id_rsa
" -N -n localhost >gpg-proxy.ssh.log 2>&1 &
315 echo $! > "${WORKDIR}/gpg-proxy.
ssh.pid
"
317 # Note that on linux we always directly mount the gpg agent's extra socket to limit what the
318 # container can ask the gpg-agent to do.
319 # When working on a remote linux machine you should be sure to forward both the remote machine's
320 # agent socket and agent extra socket to your local gpg-agent's extra socket. See the README.txt
322 GPG_PROXY_MOUNT=(--mount \
323 "type=bind,src
=$
(gpgconf
--list-dir agent-extra-socket
),dst
=/home
/${USER}/.gnupg
/S.gpg-agent
")
326 banner "Building
$RELEASE_TAG; output will be
at $WORKDIR/output
"
327 log "We should clean the container up when we are
done. If that fails
then the container ID
" \
330 # Where possible we specify "consistency
=delegated
" when we do not need host access during the
331 # build run. On Mac OS X specifically this gets us a big perf improvement.
332 cmd=(docker run --rm -ti \
333 --env-file "$ENVFILE" \
334 --cidfile "${WORKDIR}/release.cid
" \
335 --mount "type=bind,src
=${WORKDIR},dst
=/home
/${USER}/hbase-rm
,consistency
=delegated
" \
337 "${GIT_REPO_MOUNT[@]}" \
338 "${GPG_PROXY_MOUNT[@]}" \
339 "org.apache.hbase
/hbase-rm
:$IMGTAG")