dev-support/create-release/release-build.sh

   1 #!/usr/bin/env bash
   2
   3 #
   4 # Licensed to the Apache Software Foundation (ASF) under one or more
   5 # contributor license agreements.  See the NOTICE file distributed with
   6 # this work for additional information regarding copyright ownership.
   7 # The ASF licenses this file to You under the Apache License, Version 2.0
   8 # (the "License"); you may not use this file except in compliance with
   9 # the License.  You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 trap cleanup EXIT
  21
  22 # Source in utils.
  23 SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  24 # shellcheck source=SCRIPTDIR/release-util.sh
  25 . "$SELF/release-util.sh"
  26
  27 # Print usage and exit.
  28 function exit_with_usage {
  29   cat <<'EOF'
  30 Usage: release-build.sh <tag|publish-dist|publish-snapshot|publish-release>
  31 Creates release deliverables from a tag or commit.
  32 Argument: one of 'tag', 'publish-dist', 'publish-snapshot', or 'publish-release'
  33   tag               Prepares for release on specified git branch: Set release version,
  34                     update CHANGES and RELEASENOTES, create release tag,
  35                     increment version for ongoing dev, and publish to Apache git repo.
  36   publish-dist      Build and publish distribution packages (tarballs) to Apache dist repo
  37   publish-snapshot  Build and publish maven artifacts snapshot release to Apache snapshots repo
  38   publish-release   Build and publish maven artifacts release to Apache release repo, and
  39                     construct vote email from template
  40
  41 All other inputs are environment variables.  Please use do-release-docker.sh or
  42 do-release.sh to set up the needed environment variables.  This script, release-build.sh,
  43 is not intended to be called stand-alone, and such use is untested.  The env variables used are:
  44
  45 Used for 'tag' and 'publish' stages:
  46   PROJECT - The project to build. No default.
  47   RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2)
  48     Required for 'tag'; defaults for 'publish' to the version in pom at GIT_REF
  49   RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0), also used by
  50     publish-dist as package version name in dist directory path
  51   ASF_USERNAME - Username of ASF committer account
  52   ASF_PASSWORD - Password of ASF committer account
  53   DRY_RUN - 1:true (default), 0:false. If "1", does almost all the work, but doesn't actually
  54     publish anything to upstream source or object repositories. It defaults to "1", so if you want
  55     to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh.
  56
  57 Used only for 'tag':
  58   YETUS_HOME - installation location for Apache Yetus
  59   GIT_NAME - Name to use with git
  60   GIT_EMAIL - E-mail address to use with git
  61   GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch.
  62   NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT)
  63
  64 Used only for 'publish':
  65   GIT_REF - Release tag or commit to build from (defaults to $RELEASE_TAG; only need to
  66     separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time)
  67     If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master.
  68   GPG_KEY - GPG key id (usually email addr) used to sign release artifacts
  69   REPO - Set to full path of a directory to use as maven local repo (dependencies cache)
  70     to avoid re-downloading dependencies for each stage.  It is automatically set if you
  71     request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh.
  72
  73 For example:
  74  $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist
  75 EOF
  76   exit 1
  77 }
  78
  79 set -e
  80
  81 function cleanup {
  82   # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out.
  83   if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then
  84     log "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. Set REPO to reuse downloads." >&2
  85     rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true
  86     rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true
  87   fi
  88 }
  89
  90 if [ $# -ne 1 ]; then
  91   exit_with_usage
  92 fi
  93
  94 if [[ "$1" == "-h" ]]; then
  95   exit_with_usage
  96 fi
  97
  98 if [[ "$*" == *"help"* ]]; then
  99   exit_with_usage
 100 fi
 101
 102 init_locale
 103 init_java
 104 init_mvn
 105 init_python
 106 # Print out subset of perl version (used in git hooks and japi-compliance-checker)
 107 perl --version | grep 'This is'
 108
 109 rm -rf "${PROJECT}"
 110
 111 if is_debug; then
 112   set -x  # detailed logging during action
 113 fi
 114
 115 if [[ "$1" == "tag" ]]; then
 116   init_yetus
 117   # for 'tag' stage
 118   set -o pipefail
 119   check_get_passwords ASF_PASSWORD
 120   check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
 121   if [ -z "${GIT_REPO}" ]; then
 122     check_needed_vars ASF_USERNAME ASF_PASSWORD
 123   fi
 124   git_clone_overwrite
 125
 126   # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
 127   # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
 128   # projects share the same HBASE jira name.  To make this work, by convention, the HBASE jira "Fix Version"
 129   # field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0".
 130   # So, here we prepend the project name to the version, but only for the hbase sub-projects.
 131   jira_fix_version="${RELEASE_VERSION}"
 132   shopt -s nocasematch
 133   if [[ "${PROJECT}" =~ ^hbase- ]]; then
 134     jira_fix_version="${PROJECT}-${RELEASE_VERSION}"
 135   fi
 136   shopt -u nocasematch
 137   update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}"
 138
 139   cd "${PROJECT}"
 140
 141   git config user.name "$GIT_NAME"
 142   git config user.email "$GIT_EMAIL"
 143   git config user.signingkey "${GPG_KEY}"
 144
 145   # Create release version
 146   maven_set_version "$RELEASE_VERSION"
 147   find . -name pom.xml -exec git add {} \;
 148   git add RELEASENOTES.md CHANGES.md
 149
 150   git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
 151   log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH"
 152   git tag -s -m "Via create-release" "$RELEASE_TAG"
 153
 154   # Create next version
 155   maven_set_version "$NEXT_VERSION"
 156   find . -name pom.xml -exec git add {} \;
 157   git commit -s -m "Preparing development version $NEXT_VERSION"
 158
 159   if ! is_dry_run; then
 160     # Push changes
 161     git push origin "$RELEASE_TAG"
 162     git push origin "HEAD:$GIT_BRANCH"
 163     cd ..
 164     rm -rf "${PROJECT}"
 165   else
 166     cd ..
 167     mv "${PROJECT}" "${PROJECT}.tag"
 168     log "Dry run: Clone with version changes and tag available as ${PROJECT}.tag in the output directory."
 169   fi
 170   exit 0
 171 fi
 172
 173 ### Below is for 'publish-*' stages ###
 174 check_get_passwords ASF_PASSWORD
 175 check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY
 176
 177 # Commit ref to checkout when building
 178 BASE_DIR=$(pwd)
 179 GIT_REF=${GIT_REF:-master}
 180 if [[ "$PROJECT" =~ ^hbase ]]; then
 181   RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase"
 182 else
 183   RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}"
 184 fi
 185
 186 # in case of dry run, enable publish steps to chain from tag step
 187 if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
 188   ln -s "${PROJECT}.tag" "${PROJECT}"
 189 else
 190   git_clone_overwrite
 191 fi
 192 cd "${PROJECT}"
 193 git checkout "$GIT_REF"
 194 git_hash="$(git rev-parse --short HEAD)"
 195 GIT_LONG_HASH="$(git rev-parse HEAD)"
 196 log "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash"
 197
 198 if [ -z "${RELEASE_VERSION}" ]; then
 199   RELEASE_VERSION="$(maven_get_version)"
 200 fi
 201
 202 # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins
 203 # machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion
 204 # in its pull request.
 205 LSOF=lsof
 206 if ! hash $LSOF 2>/dev/null; then
 207   LSOF=/usr/sbin/lsof
 208 fi
 209
 210 package_version_name="$RELEASE_TAG"
 211 if [ -z "$package_version_name" ]; then
 212   package_version_name="${RELEASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}"
 213 fi
 214
 215 git clean -d -f -x
 216 cd ..
 217 if [[ "$PROJECT" =~ ^hbase- ]]; then
 218   DEST_DIR_NAME="${PROJECT}-${package_version_name}"
 219 else
 220   DEST_DIR_NAME="$package_version_name"
 221 fi
 222
 223 if [[ "$1" == "publish-dist" ]]; then
 224   # Source and binary tarballs
 225   log "Packaging release source tarballs"
 226   make_src_release "${PROJECT}" "${RELEASE_VERSION}"
 227
 228   # we do not have binary tarballs for hbase-thirdparty
 229   if [[ "${PROJECT}" != "hbase-thirdparty" ]]; then
 230     make_binary_release "${PROJECT}" "${RELEASE_VERSION}"
 231   fi
 232
 233   svn_target="svn-${PROJECT}"
 234   svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target"
 235   rm -rf "${svn_target:?}/${DEST_DIR_NAME}"
 236   mkdir -p "$svn_target/${DEST_DIR_NAME}"
 237
 238   log "Copying release tarballs"
 239   cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/"
 240   cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
 241   cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"
 242   shopt -s nocasematch
 243   # Generate api report only if project is hbase for now.
 244   if [ "${PROJECT}" == "hbase" ]; then
 245     # This script usually reports an errcode along w/ the report.
 246     generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${GIT_REF}" || true
 247     cp api*.html "$svn_target/${DEST_DIR_NAME}/"
 248   fi
 249   shopt -u nocasematch
 250
 251   log "svn add"
 252   svn add "$svn_target/${DEST_DIR_NAME}"
 253
 254   if ! is_dry_run; then
 255     cd "$svn_target"
 256     svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $package_version_name" --no-auth-cache
 257     cd ..
 258     rm -rf "$svn_target"
 259   else
 260     mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist"
 261     log "Dry run: svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist"
 262     rm -rf "$svn_target"
 263   fi
 264   log "svn ci done"
 265
 266   exit 0
 267 fi
 268
 269 if [[ "$1" == "publish-snapshot" ]]; then
 270   (
 271   cd "${PROJECT}"
 272   mvn_log="${BASE_DIR}/mvn_deploy_snapshot.log"
 273   log "Publishing snapshot to nexus"
 274   maven_deploy snapshot "$mvn_log"
 275   if ! is_dry_run; then
 276     log "Snapshot artifacts successfully published to repo."
 277     rm "$mvn_log"
 278   else
 279     log "Dry run: Snapshot artifacts successfully built, but not published due to dry run."
 280   fi
 281   )
 282   exit $?
 283 fi
 284
 285 if [[ "$1" == "publish-release" ]]; then
 286   (
 287   cd "${PROJECT}"
 288   mvn_log="${BASE_DIR}/mvn_deploy_release.log"
 289   log "Staging release in nexus"
 290   maven_deploy release "$mvn_log"
 291   declare staged_repo_id="dryrun-no-repo"
 292   if ! is_dry_run; then
 293     staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \
 294         | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/')
 295     log "Release artifacts successfully published to repo ${staged_repo_id}"
 296     rm "$mvn_log"
 297   else
 298     log "Dry run: Release artifacts successfully built, but not published due to dry run."
 299   fi
 300   # Dump out email to send. Where we find vote.tmpl depends
 301   # on where this script is run from
 302   PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g'
 303   export PROJECT_TEXT
 304   eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt"
 305   )
 306   exit $?
 307 fi
 308
 309 set +x  # done with detailed logging
 310 cd ..
 311 rm -rf "${PROJECT}"
 312 log "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2
 313 exit_with_usage