dev-support/create-release/do-release-docker.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Licensed to the Apache Software Foundation (ASF) under one or more
   4 # contributor license agreements.  See the NOTICE file distributed with
   5 # this work for additional information regarding copyright ownership.
   6 # The ASF licenses this file to You under the Apache License, Version 2.0
   7 # (the "License"); you may not use this file except in compliance with
   8 # the License.  You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #
  18
  19 #
  20 # Creates a HBase release candidate. The script will update versions, tag the branch,
  21 # build HBase binary packages and documentation, and upload maven artifacts to a staging
  22 # repository. There is also a dry run mode where only local builds are performed, and
  23 # nothing is uploaded to the ASF repos.
  24 #
  25 # Run with "-h" for options. For example, running below will do all
  26 # steps above using the 'rm' dir under Downloads as workspace:
  27 #
  28 # $ ./do-release-docker.sh  -d ~/Downloads/rm
  29 #
  30 # The scripts in this directory came originally from spark [1]. They were then
  31 # modified to suite the hbase context. These scripts supercedes the old
  32 # ../make_rc.sh script for making release candidates because what is here is more
  33 # comprehensive doing more steps of the RM process as well as running in a
  34 # container so the RM build environment can be a constant.
  35 #
  36 # It:
  37 #  * Tags release
  38 #  * Sets version to the release version
  39 #  * Sets version to next SNAPSHOT version.
  40 #  * Builds, signs, and hashes all artifacts.
  41 #  * Pushes release tgzs to the dev dir in a apache dist.
  42 #  * Pushes to repository.apache.org staging.
  43 #
  44 # The entry point is here, in the do-release-docker.sh script.
  45 #
  46 # 1. https://github.com/apache/spark/tree/master/dev/create-release
  47 #
  48 set -e
  49
  50 # Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools
  51 export PROJECT="${PROJECT:-hbase}"
  52
  53 SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  54 # shellcheck source=SCRIPTDIR/release-util.sh
  55 . "$SELF/release-util.sh"
  56 ORIG_PWD="$(pwd)"
  57
  58 function usage {
  59   local NAME
  60   NAME="$(basename "${BASH_SOURCE[0]}")"
  61   cat <<EOF
  62 Usage: $NAME [options]
  63
  64 This script runs the release scripts inside a docker image.
  65
  66 Options:
  67
  68   -d [path]    required. working directory. output will be written to "output" in here.
  69   -f           "force" -- actually publish this release. Unless you specify '-f', it will
  70                default to dry run mode, which checks and does local builds, but does not upload anything.
  71   -t [tag]     tag for the hbase-rm docker image to use for building (default: "latest").
  72   -j [path]    path to local JDK installation to use building. By default the script will
  73                use openjdk8 installed in the docker image.
  74   -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
  75   -r [repo]    git repo to use for remote git operations. defaults to ASF gitbox for project.
  76   -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
  77                If none specified, runs tag, then publish-dist, and then publish-release.
  78                'publish-snapshot' is also an allowed, less used, option.
  79 EOF
  80   exit 1
  81 }
  82
  83 WORKDIR=
  84 IMGTAG=latest
  85 JAVA=
  86 RELEASE_STEP=
  87 GIT_REPO=
  88 while getopts "d:fhj:p:r:s:t:" opt; do
  89   case $opt in
  90     d) WORKDIR="$OPTARG" ;;
  91     f) DRY_RUN=0 ;;
  92     t) IMGTAG="$OPTARG" ;;
  93     j) JAVA="$OPTARG" ;;
  94     p) PROJECT="$OPTARG" ;;
  95     r) GIT_REPO="$OPTARG" ;;
  96     s) RELEASE_STEP="$OPTARG" ;;
  97     h) usage ;;
  98     ?) error "Invalid option. Run with -h for help." ;;
  99   esac
 100 done
 101 shift $((OPTIND-1))
 102 if (( $# > 0 )); then
 103   error "Arguments can only be provided with option flags, invalid args: $*"
 104 fi
 105
 106 if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
 107   error "Work directory (-d) must be defined and exist. Run with -h for help."
 108 fi
 109
 110 if [ -d "$WORKDIR/output" ]; then
 111   read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
 112   if [ "$ANSWER" != "y" ]; then
 113     error "Exiting."
 114   fi
 115 fi
 116
 117 cd "$WORKDIR"
 118 rm -rf "$WORKDIR/output"
 119 mkdir "$WORKDIR/output"
 120
 121 get_release_info
 122
 123 # Place all RM scripts and necessary data in a local directory that must be defined in the command
 124 # line. This directory is mounted into the image. Its WORKDIR, the arg passed with -d.
 125 for f in "$SELF"/*; do
 126   if [ -f "$f" ]; then
 127     cp "$f" "$WORKDIR"
 128   fi
 129 done
 130
 131 GPG_KEY_FILE="$WORKDIR/gpg.key"
 132 fcreate_secure "$GPG_KEY_FILE"
 133 $GPG --passphrase "$GPG_PASSPHRASE" --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
 134
 135 run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
 136   docker build -t "hbase-rm:$IMGTAG" --build-arg UID=$UID "$SELF/hbase-rm"
 137
 138 # Write the release information to a file with environment variables to be used when running the
 139 # image.
 140 ENVFILE="$WORKDIR/env.list"
 141 fcreate_secure "$ENVFILE"
 142
 143 function cleanup {
 144   rm -f "$ENVFILE"
 145   rm -f "$GPG_KEY_FILE"
 146 }
 147
 148 trap cleanup EXIT
 149
 150 cat > "$ENVFILE" <<EOF
 151 PROJECT=$PROJECT
 152 DRY_RUN=$DRY_RUN
 153 SKIP_TAG=$SKIP_TAG
 154 RUNNING_IN_DOCKER=1
 155 GIT_BRANCH=$GIT_BRANCH
 156 NEXT_VERSION=$NEXT_VERSION
 157 RELEASE_VERSION=$RELEASE_VERSION
 158 RELEASE_TAG=$RELEASE_TAG
 159 GIT_REF=$GIT_REF
 160 ASF_USERNAME=$ASF_USERNAME
 161 GIT_NAME=$GIT_NAME
 162 GIT_EMAIL=$GIT_EMAIL
 163 GPG_KEY=$GPG_KEY
 164 ASF_PASSWORD=$ASF_PASSWORD
 165 GPG_PASSPHRASE=$GPG_PASSPHRASE
 166 RELEASE_STEP=$RELEASE_STEP
 167 API_DIFF_TAG=$API_DIFF_TAG
 168 EOF
 169
 170 JAVA_VOL=()
 171 if [ -n "$JAVA" ]; then
 172   echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
 173   JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
 174 fi
 175
 176 #TODO some debug output would be good here
 177 GIT_REPO_MOUNT=()
 178 if [ -n "${GIT_REPO}" ]; then
 179   case "${GIT_REPO}" in
 180     # skip the easy to identify remote protocols
 181     ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
 182     # for sure local
 183     /*)
 184       GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
 185       echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 186       GIT_REPO="/opt/hbase-repo"
 187       ;;
 188     # on the host but normally git wouldn't use the local optimization
 189     file://*)
 190       echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
 191       GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
 192       echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 193       GIT_REPO="/opt/hbase-repo"
 194       ;;
 195     # have to decide if it's a local path or the "scp-ish" remote
 196     *)
 197       declare colon_remove_prefix;
 198       declare slash_remove_prefix;
 199       declare local_path;
 200       colon_remove_prefix="${GIT_REPO#*:}"
 201       slash_remove_prefix="${GIT_REPO#*/}"
 202       if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
 203         # if there was no colon at all, we assume this must be a local path
 204         local_path="no colon at all"
 205       elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
 206         # if there was a colon and there is no slash, then we assume it must be scp-style host
 207         # and a relative path
 208
 209         if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
 210           # Given the substrings made by removing everything up to the first colon and slash
 211           # we can determine which comes first based on the longer substring length.
 212           # if the slash is first, then we assume the colon is part of a path name and if the colon
 213           # is first then it is the seperator between a scp-style host name and the path.
 214           local_path="slash happened before a colon"
 215         fi
 216       fi
 217       if [ -n "${local_path}" ]; then
 218         # convert to an absolute path
 219         GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
 220         GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
 221         echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 222         GIT_REPO="/opt/hbase-repo"
 223       fi
 224       ;;
 225   esac
 226   echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 227 fi
 228
 229 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
 230 cmd=(docker run -ti \
 231   --env-file "$ENVFILE" \
 232   --volume "$WORKDIR:/opt/hbase-rm" \
 233   "${JAVA_VOL[@]}" \
 234   "${GIT_REPO_MOUNT[@]}" \
 235   "hbase-rm:$IMGTAG")
 236 echo "${cmd[*]}"
 237 "${cmd[@]}"