dev-support/create-release/do-release-docker.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Licensed to the Apache Software Foundation (ASF) under one or more
   4 # contributor license agreements.  See the NOTICE file distributed with
   5 # this work for additional information regarding copyright ownership.
   6 # The ASF licenses this file to You under the Apache License, Version 2.0
   7 # (the "License"); you may not use this file except in compliance with
   8 # the License.  You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #
  18
  19 #
  20 # Creates a HBase release candidate. The script will update versions, tag the branch,
  21 # build HBase binary packages and documentation, and upload maven artifacts to a staging
  22 # repository. There is also a dry run mode where only local builds are performed, and
  23 # nothing is uploaded to the ASF repos.
  24 #
  25 # Run with "-h" for options. For example, running below will do all
  26 # steps above using the 'rm' dir under Downloads as workspace:
  27 #
  28 # $ ./do-release-docker.sh  -d ~/Downloads/rm
  29 #
  30 # The scripts in this directory came originally from spark [1]. They were then
  31 # modified to suite the hbase context. These scripts supercedes the old
  32 # ../make_rc.sh script for making release candidates because what is here is more
  33 # comprehensive doing more steps of the RM process as well as running in a
  34 # container so the RM build environment can be a constant.
  35 #
  36 # It:
  37 #  * Tags release
  38 #  * Sets version to the release version
  39 #  * Sets version to next SNAPSHOT version.
  40 #  * Builds, signs, and hashes all artifacts.
  41 #  * Pushes release tgzs to the dev dir in a apache dist.
  42 #  * Pushes to repository.apache.org staging.
  43 #
  44 # The entry point is here, in the do-release-docker.sh script.
  45 #
  46 # 1. https://github.com/apache/spark/tree/master/dev/create-release
  47 #
  48 set -e
  49
  50 # Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools
  51 export PROJECT="${PROJECT:-hbase}"
  52
  53 SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  54 # shellcheck source=SCRIPTDIR/release-util.sh
  55 . "$SELF/release-util.sh"
  56 ORIG_PWD="$(pwd)"
  57
  58 function usage {
  59   local NAME
  60   NAME="$(basename "${BASH_SOURCE[0]}")"
  61   cat <<EOF
  62 Usage: $NAME [options]
  63
  64 This script runs the release scripts inside a docker image.
  65
  66 Options:
  67
  68   -d [path]    required. working directory. output will be written to "output" in here.
  69   -f           "force" -- actually publish this release. Unless you specify '-f', it will
  70                default to dry run mode, which checks and does local builds, but does not upload anything.
  71   -t [tag]     tag for the hbase-rm docker image to use for building (default: "latest").
  72   -j [path]    path to local JDK installation to use building. By default the script will
  73                use openjdk8 installed in the docker image.
  74   -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
  75   -r [repo]    git repo to use for remote git operations. defaults to ASF gitbox for project.
  76   -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
  77                If none specified, runs tag, then publish-dist, and then publish-release.
  78                'publish-snapshot' is also an allowed, less used, option.
  79   -x           debug. Does less clean up (env file, gpg forwarding on mac)
  80 EOF
  81   exit 1
  82 }
  83
  84 WORKDIR=
  85 IMGTAG=latest
  86 JAVA=
  87 RELEASE_STEP=
  88 GIT_REPO=
  89 while getopts "d:fhj:p:r:s:t:x" opt; do
  90   case $opt in
  91     d) WORKDIR="$OPTARG" ;;
  92     f) DRY_RUN=0 ;;
  93     t) IMGTAG="$OPTARG" ;;
  94     j) JAVA="$OPTARG" ;;
  95     p) PROJECT="$OPTARG" ;;
  96     r) GIT_REPO="$OPTARG" ;;
  97     s) RELEASE_STEP="$OPTARG" ;;
  98     x) DEBUG=1 ;;
  99     h) usage ;;
 100     ?) error "Invalid option. Run with -h for help." ;;
 101   esac
 102 done
 103 shift $((OPTIND-1))
 104 if (( $# > 0 )); then
 105   error "Arguments can only be provided with option flags, invalid args: $*"
 106 fi
 107 export DEBUG
 108
 109 if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
 110   error "Work directory (-d) must be defined and exist. Run with -h for help."
 111 fi
 112
 113 if [ -d "$WORKDIR/output" ]; then
 114   read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
 115   if [ "$ANSWER" != "y" ]; then
 116     error "Exiting."
 117   fi
 118 fi
 119
 120 if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ] || \
 121    [ -f "${WORKDIR}/gpg-proxy.cid" ] || \
 122    [ -f "${WORKDIR}/release.cid" ]; then
 123   read -r -p "container/pid files from prior run exists. Overwrite and continue? [y/n] " ANSWER
 124   if [ "$ANSWER" != "y" ]; then
 125     error "Exiting."
 126   fi
 127 fi
 128
 129 cd "$WORKDIR"
 130 rm -rf "$WORKDIR/output"
 131 rm -rf "${WORKDIR}/gpg-proxy.ssh.pid" "${WORKDIR}/gpg-proxy.cid" "${WORKDIR}/release.cid"
 132 mkdir "$WORKDIR/output"
 133
 134 banner "Gathering release details."
 135 HOST_OS="$(get_host_os)"
 136 get_release_info
 137
 138 banner "Setup"
 139
 140 # Place all RM scripts and necessary data in a local directory that must be defined in the command
 141 # line. This directory is mounted into the image. Its WORKDIR, the arg passed with -d.
 142 for f in "$SELF"/*; do
 143   if [ -f "$f" ]; then
 144     cp "$f" "$WORKDIR"
 145   fi
 146 done
 147
 148 # We need to import that public key in the container in order to use the private key via the agent.
 149 GPG_KEY_FILE="$WORKDIR/gpg.key.public"
 150 log "Exporting public key for ${GPG_KEY}"
 151 fcreate_secure "$GPG_KEY_FILE"
 152 $GPG "${GPG_ARGS[@]}" --export "${GPG_KEY}" > "${GPG_KEY_FILE}"
 153
 154 function cleanup {
 155   local id
 156   banner "Release Cleanup"
 157   if is_debug; then
 158     log "skipping due to debug run"
 159     return 0
 160   fi
 161   log "details in cleanup.log"
 162   if [ -f "${ENVFILE}" ]; then
 163     rm -f "$ENVFILE"
 164   fi
 165   rm -f "$GPG_KEY_FILE"
 166   if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ]; then
 167     id=$(cat "${WORKDIR}/gpg-proxy.ssh.pid")
 168     echo "Stopping ssh tunnel for gpg-agent at PID ${id}" | tee -a cleanup.log
 169     kill -9 "${id}" >>cleanup.log 2>&1 || true
 170     rm -f "${WORKDIR}/gpg-proxy.ssh.pid" >>cleanup.log 2>&1
 171   fi
 172   if [ -f "${WORKDIR}/gpg-proxy.cid" ]; then
 173     id=$(cat "${WORKDIR}/gpg-proxy.cid")
 174     echo "Stopping gpg-proxy container with ID ${id}" | tee -a cleanup.log
 175     docker kill "${id}" >>cleanup.log 2>&1 || true
 176     rm -f "${WORKDIR}/gpg-proxy.cid" >>cleanup.log 2>&1
 177     # TODO we should remove the gpgagent volume?
 178   fi
 179   if [ -f "${WORKDIR}/release.cid" ]; then
 180     id=$(cat "${WORKDIR}/release.cid")
 181     echo "Stopping release container with ID ${id}" | tee -a cleanup.log
 182     docker kill "${id}" >>cleanup.log 2>&1 || true
 183     rm -f "${WORKDIR}/release.cid" >>cleanup.log 2>&1
 184   fi
 185 }
 186
 187 trap cleanup EXIT
 188
 189 log "Host OS: ${HOST_OS}"
 190 if [ "${HOST_OS}" == "DARWIN" ]; then
 191   run_silent "Building gpg-agent-proxy image with tag ${IMGTAG}..." "docker-proxy-build.log" \
 192     docker build --build-arg "UID=${UID}" --build-arg "RM_USER=${USER}" \
 193         --tag "org.apache.hbase/gpg-agent-proxy:${IMGTAG}" "${SELF}/mac-sshd-gpg-agent"
 194 fi
 195
 196 run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
 197   docker build --tag "org.apache.hbase/hbase-rm:$IMGTAG" --build-arg "UID=$UID" \
 198       --build-arg "RM_USER=${USER}" "$SELF/hbase-rm"
 199
 200 banner "Final prep for container launch."
 201 log "Writing out environment for container."
 202 # Write the release information to a file with environment variables to be used when running the
 203 # image.
 204 ENVFILE="$WORKDIR/env.list"
 205 fcreate_secure "$ENVFILE"
 206
 207 cat > "$ENVFILE" <<EOF
 208 PROJECT=$PROJECT
 209 DRY_RUN=$DRY_RUN
 210 SKIP_TAG=$SKIP_TAG
 211 RUNNING_IN_DOCKER=1
 212 GIT_BRANCH=$GIT_BRANCH
 213 NEXT_VERSION=$NEXT_VERSION
 214 RELEASE_VERSION=$RELEASE_VERSION
 215 RELEASE_TAG=$RELEASE_TAG
 216 GIT_REF=$GIT_REF
 217 ASF_USERNAME=$ASF_USERNAME
 218 GIT_NAME=$GIT_NAME
 219 GIT_EMAIL=$GIT_EMAIL
 220 GPG_KEY=$GPG_KEY
 221 ASF_PASSWORD=$ASF_PASSWORD
 222 RELEASE_STEP=$RELEASE_STEP
 223 API_DIFF_TAG=$API_DIFF_TAG
 224 HOST_OS=$HOST_OS
 225 EOF
 226
 227 JAVA_MOUNT=()
 228 if [ -n "$JAVA" ]; then
 229   echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
 230   JAVA_MOUNT=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly")
 231 fi
 232
 233 #TODO some debug output would be good here
 234 GIT_REPO_MOUNT=()
 235 if [ -n "${GIT_REPO}" ]; then
 236   case "${GIT_REPO}" in
 237     # skip the easy to identify remote protocols
 238     ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
 239     # for sure local
 240     /*)
 241       GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo,consistency=delegated")
 242       echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 243       GIT_REPO="/opt/hbase-repo"
 244       ;;
 245     # on the host but normally git wouldn't use the local optimization
 246     file://*)
 247       log "Converted file:// git repo to a local path, which changes git to assume --local."
 248       GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo,consistency=delegated")
 249       echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 250       GIT_REPO="/opt/hbase-repo"
 251       ;;
 252     # have to decide if it's a local path or the "scp-ish" remote
 253     *)
 254       declare colon_remove_prefix;
 255       declare slash_remove_prefix;
 256       declare local_path;
 257       colon_remove_prefix="${GIT_REPO#*:}"
 258       slash_remove_prefix="${GIT_REPO#*/}"
 259       if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
 260         # if there was no colon at all, we assume this must be a local path
 261         local_path="no colon at all"
 262       elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
 263         # if there was a colon and there is no slash, then we assume it must be scp-style host
 264         # and a relative path
 265
 266         if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
 267           # Given the substrings made by removing everything up to the first colon and slash
 268           # we can determine which comes first based on the longer substring length.
 269           # if the slash is first, then we assume the colon is part of a path name and if the colon
 270           # is first then it is the seperator between a scp-style host name and the path.
 271           local_path="slash happened before a colon"
 272         fi
 273       fi
 274       if [ -n "${local_path}" ]; then
 275         # convert to an absolute path
 276         GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
 277         GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo,consistency=delegated")
 278         echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 279         GIT_REPO="/opt/hbase-repo"
 280       fi
 281       ;;
 282   esac
 283   echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 284 fi
 285
 286 GPG_PROXY_MOUNT=()
 287 if [ "${HOST_OS}" == "DARWIN" ]; then
 288   GPG_PROXY_MOUNT=(--mount "type=volume,src=gpgagent,dst=/home/${USER}/.gnupg/")
 289   log "Setting up GPG agent proxy container needed on OS X."
 290   log "    we should clean this up for you. If that fails the container ID is below and in " \
 291       "gpg-proxy.cid"
 292   #TODO the key pair used should be configurable
 293   docker run --rm -p 62222:22 \
 294      --detach --cidfile "${WORKDIR}/gpg-proxy.cid" \
 295      --mount \
 296      "type=bind,src=${HOME}/.ssh/id_rsa.pub,dst=/home/${USER}/.ssh/authorized_keys,readonly" \
 297      "${GPG_PROXY_MOUNT[@]}" \
 298      "org.apache.hbase/gpg-agent-proxy:${IMGTAG}"
 299   # gotta trust the container host
 300   ssh-keyscan -p 62222 localhost 2>/dev/null | sort > "${WORKDIR}/gpg-agent-proxy.ssh-keyscan"
 301   sort "${HOME}/.ssh/known_hosts" | comm -1 -3 - "${WORKDIR}/gpg-agent-proxy.ssh-keyscan" \
 302       > "${WORKDIR}/gpg-agent-proxy.known_hosts"
 303   if [ -s "${WORKDIR}/gpg-agent-proxy.known_hosts" ]; then
 304     log "Your ssh known_hosts does not include the entries for the gpg-agent proxy container."
 305     log "The following entry(ies) are missing:"
 306     sed -e 's/^/    /' "${WORKDIR}/gpg-agent-proxy.known_hosts"
 307     read -r -p "Okay to add these entries to ${HOME}/.ssh/known_hosts? [y/n] " ANSWER
 308     if [ "$ANSWER" != "y" ]; then
 309       error "Exiting."
 310     fi
 311     cat "${WORKDIR}/gpg-agent-proxy.known_hosts" >> "${HOME}/.ssh/known_hosts"
 312   fi
 313   log "Launching ssh reverse tunnel from the container to gpg agent."
 314   log "    we should clean this up for you. If that fails the PID is in gpg-proxy.ssh.pid"
 315   ssh -p 62222 -R "/home/${USER}/.gnupg/S.gpg-agent:$(gpgconf --list-dir agent-extra-socket)" \
 316       -i "${HOME}/.ssh/id_rsa" -N -n localhost >gpg-proxy.ssh.log 2>&1 &
 317   echo $! > "${WORKDIR}/gpg-proxy.ssh.pid"
 318 else
 319   # Note that on linux we always directly mount the gpg agent's extra socket to limit what the
 320   # container can ask the gpg-agent to do.
 321   # When working on a remote linux machine you should be sure to forward both the remote machine's
 322   # agent socket and agent extra socket to your local gpg-agent's extra socket. See the README.txt
 323   # for an example.
 324   GPG_PROXY_MOUNT=(--mount \
 325       "type=bind,src=$(gpgconf --list-dir agent-extra-socket),dst=/home/${USER}/.gnupg/S.gpg-agent")
 326 fi
 327
 328 banner "Building $RELEASE_TAG; output will be at $WORKDIR/output"
 329 log "We should clean the container up when we are done. If that fails then the container ID " \
 330     "is in release.cid"
 331 echo
 332 # Where possible we specify "consistency=delegated" when we do not need host access during the
 333 # build run. On Mac OS X specifically this gets us a big perf improvement.
 334 cmd=(docker run --rm -ti \
 335   --env-file "$ENVFILE" \
 336   --cidfile "${WORKDIR}/release.cid" \
 337   --mount "type=bind,src=${WORKDIR},dst=/home/${USER}/hbase-rm,consistency=delegated" \
 338   "${JAVA_MOUNT[@]}" \
 339   "${GIT_REPO_MOUNT[@]}" \
 340   "${GPG_PROXY_MOUNT[@]}" \
 341   "org.apache.hbase/hbase-rm:$IMGTAG")
 342 echo "${cmd[*]}"
 343 "${cmd[@]}"