dev-support/create-release/release-util.sh

   1 #!/usr/bin/env bash
   2
   3 #
   4 # Licensed to the Apache Software Foundation (ASF) under one or more
   5 # contributor license agreements.  See the NOTICE file distributed with
   6 # this work for additional information regarding copyright ownership.
   7 # The ASF licenses this file to You under the Apache License, Version 2.0
   8 # (the "License"); you may not use this file except in compliance with
   9 # the License.  You may obtain a copy of the License at
  10 #
  11 #    http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 # Source this file if you want to use any of its utiilty (also useful
  21 # testing the below functions). Do "$ . ./release-util.sh" and then
  22 # you can do stuff like call the CHANGES updating function
  23 # update_releasenotes:
  24 #
  25 #  $ update_releasenotes ~/checkouts/hbase.apache.git 2.3.4
  26 #
  27 # Just make sure any environment variables needed are predefined
  28 # in your context.
  29 #
  30 DRY_RUN=${DRY_RUN:-1} #default to dry run
  31 DEBUG=${DEBUG:-0}
  32 GPG=${GPG:-gpg}
  33 GPG_ARGS=(--no-autostart --batch)
  34 if [ -n "${GPG_KEY}" ]; then
  35   GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}")
  36 fi
  37 # Maven Profiles for publishing snapshots and release to Maven Central and Dist
  38 PUBLISH_PROFILES=("-P" "apache-release,release")
  39
  40 function error {
  41   log "Error: $*" >&2
  42   exit 1
  43 }
  44
  45 function read_config {
  46   local PROMPT="$1"
  47   local DEFAULT="$2"
  48   local REPLY=
  49
  50   read -r -p "$PROMPT [$DEFAULT]: " REPLY
  51   local RETVAL="${REPLY:-$DEFAULT}"
  52   if [ -z "$RETVAL" ]; then
  53     error "$PROMPT must be provided."
  54   fi
  55   echo "$RETVAL"
  56 }
  57
  58 function parse_version {
  59   grep -e '<version>.*</version>' | \
  60     head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
  61 }
  62
  63 function banner {
  64   local msg="$1"
  65   echo "========================"
  66   log "${msg}"
  67   echo
  68 }
  69
  70 function log {
  71   echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ${1}"
  72 }
  73
  74 # current number of seconds since epoch
  75 function get_ctime {
  76   date +"%s"
  77 }
  78
  79 function run_silent {
  80   local BANNER="$1"
  81   local LOG_FILE="$2"
  82   shift 2
  83   local -i start_time
  84   local -i stop_time
  85
  86   banner "${BANNER}"
  87   log "Command: $*"
  88   log "Log file: $LOG_FILE"
  89   start_time="$(get_ctime)"
  90
  91   if ! "$@" 1>"$LOG_FILE" 2>&1; then
  92     log "Command FAILED. Check full logs for details."
  93     tail "$LOG_FILE"
  94     exit 1
  95   fi
  96   stop_time="$(get_ctime)"
  97   log "SUCCESS ($((stop_time - start_time)) seconds)"
  98 }
  99
 100 function fcreate_secure {
 101   local FPATH="$1"
 102   rm -f "$FPATH"
 103   touch "$FPATH"
 104   chmod 600 "$FPATH"
 105 }
 106
 107 # API compare version.
 108 function get_api_diff_version {
 109   local version="$1"
 110   local rev
 111   local api_diff_tag
 112   rev=$(echo "$version" | cut -d . -f 3)
 113   if [ "$rev" != 0 ]; then
 114     local short_version
 115     short_version="$(echo "$version" | cut -d . -f 1-2)"
 116     api_diff_tag="rel/${short_version}.$((rev - 1))"
 117   else
 118     local major minor
 119     major="$(echo "$version" | cut -d . -f 1)"
 120     minor="$(echo "$version" | cut -d . -f 2)"
 121     if [ "$minor" != 0 ]; then
 122       api_diff_tag="rel/${major}.$((minor - 1)).0"
 123     else
 124       api_diff_tag="rel/$((major - 1)).0.0"
 125     fi
 126   fi
 127   api_diff_tag="$(read_config "api_diff_tag" "$api_diff_tag")"
 128   echo "$api_diff_tag"
 129 }
 130
 131 # Get all branches that begin with 'branch-', the hbase convention for
 132 # release branches, sort them and then pop off the most recent.
 133 function get_release_info {
 134   PROJECT="$(read_config "PROJECT" "$PROJECT")"
 135   export PROJECT
 136
 137   if [[ -z "${ASF_REPO}" ]]; then
 138     ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git"
 139   fi
 140   if [[ -z "${ASF_REPO_WEBUI}" ]]; then
 141     ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=${PROJECT}.git"
 142   fi
 143   if [[ -z "${ASF_GITHUB_REPO}" ]]; then
 144     ASF_GITHUB_REPO="https://github.com/apache/${PROJECT}"
 145   fi
 146   if [ -z "$GIT_BRANCH" ]; then
 147     # If no branch is specified, find out the latest branch from the repo.
 148     GIT_BRANCH="$(git ls-remote --heads "$ASF_REPO" |
 149       grep refs/heads/branch- |
 150       awk '{print $2}' |
 151       sort -r |
 152       head -n 1 |
 153       cut -d/ -f3)"
 154   fi
 155
 156   GIT_BRANCH="$(read_config "GIT_BRANCH" "$GIT_BRANCH")"
 157   export GIT_BRANCH
 158
 159   # Find the current version for the branch.
 160   local version
 161   version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
 162     parse_version)"
 163   log "Current branch VERSION is $version."
 164
 165   NEXT_VERSION="$version"
 166   RELEASE_VERSION=""
 167   SHORT_VERSION="$(echo "$version" | cut -d . -f 1-2)"
 168   if [[ ! "$version" =~ .*-SNAPSHOT ]]; then
 169     RELEASE_VERSION="$version"
 170   else
 171     RELEASE_VERSION="${version/-SNAPSHOT/}"
 172   fi
 173
 174   local REV
 175   REV="$(echo "${RELEASE_VERSION}" | cut -d . -f 3)"
 176
 177   # Find out what RC is being prepared.
 178   # - If the current version is "x.y.0", then this is RC0 of the "x.y.0" release.
 179   # - If not, need to check whether the previous version has been already released or not.
 180   #   - If it has, then we're building RC0 of the current version.
 181   #   - If it has not, we're building the next RC of the previous version.
 182   if [[ -z "${RC_COUNT}" ]]; then
 183     local RC_COUNT
 184     if [ "$REV" != 0 ]; then
 185       local PREV_REL_REV=$((REV - 1))
 186       PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}"
 187       if git ls-remote --tags "$ASF_REPO" "$PREV_REL_TAG" | grep -q "refs/tags/${PREV_REL_TAG}$" ; then
 188         RC_COUNT=0
 189         REV=$((REV + 1))
 190         NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
 191       else
 192         RELEASE_VERSION="${SHORT_VERSION}.${PREV_REL_REV}"
 193         RC_COUNT="$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l)"
 194         # This makes a 'number' of it.
 195         RC_COUNT=$((RC_COUNT))
 196       fi
 197     else
 198       REV=$((REV + 1))
 199       NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
 200       RC_COUNT=0
 201     fi
 202   fi
 203
 204   RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")"
 205   NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")"
 206   export RELEASE_VERSION NEXT_VERSION
 207
 208   RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")"
 209   if [[ -z "${RELEASE_TAG}" ]]; then
 210     RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}"
 211     RELEASE_TAG="$(read_config "RELEASE_TAG" "$RELEASE_TAG")"
 212   fi
 213
 214   # Check if the RC already exists, and if re-creating the RC, skip tag creation.
 215   SKIP_TAG=0
 216   if git ls-remote --tags "$ASF_REPO" "$RELEASE_TAG" | grep -q "refs/tags/${RELEASE_TAG}$" ; then
 217     read -r -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER
 218     if [ "$ANSWER" != "y" ]; then
 219       log "Exiting."
 220       exit 1
 221     fi
 222     SKIP_TAG=1
 223   fi
 224
 225   export RELEASE_TAG SKIP_TAG
 226
 227   GIT_REF="$RELEASE_TAG"
 228   if is_dry_run; then
 229     log "This is a dry run. If tag does not actually exist, please confirm the ref that will be built for testing."
 230     GIT_REF="$(read_config "GIT_REF" "$GIT_REF")"
 231   fi
 232   export GIT_REF
 233
 234   API_DIFF_TAG="$(get_api_diff_version "$RELEASE_VERSION")"
 235
 236   # Gather some user information.
 237   ASF_USERNAME="$(read_config "ASF_USERNAME" "$LOGNAME")"
 238
 239   GIT_NAME="$(git config user.name || echo "")"
 240   GIT_NAME="$(read_config "GIT_NAME" "$GIT_NAME")"
 241
 242   GIT_EMAIL="$ASF_USERNAME@apache.org"
 243   if [[ -z "${GPG_KEY}" ]]; then
 244     GPG_KEY="$(read_config "GPG_KEY" "$GIT_EMAIL")"
 245   fi
 246   if ! GPG_KEY_ID=$("${GPG}" "${GPG_ARGS[@]}" --keyid-format 0xshort --list-public-key "${GPG_KEY}" | grep "\[S\]" | grep -o "0x[0-9A-F]*") ||
 247       [ -z "${GPG_KEY_ID}" ] ; then
 248     GPG_KEY_ID=$("${GPG}" "${GPG_ARGS[@]}" --keyid-format 0xshort --list-public-key "${GPG_KEY}" | head -n 1 | grep -o "0x[0-9A-F]*" || true)
 249   fi
 250   read -r -p "Does the GPG key '${GPG_KEY}' corresponds to the GPG key id '${GPG_KEY_ID}'. Is this correct [y/n]? " ANSWER
 251   if [ "$ANSWER" = "y" ]; then
 252     GPG_KEY="${GPG_KEY_ID}"
 253   fi
 254   export API_DIFF_TAG ASF_USERNAME GIT_NAME GIT_EMAIL GPG_KEY
 255
 256   cat <<EOF
 257 ================
 258 Release details:
 259 GIT_BRANCH:      $GIT_BRANCH
 260 RELEASE_VERSION: $RELEASE_VERSION
 261 NEXT_VERSION:    $NEXT_VERSION
 262 RELEASE_TAG:     $RELEASE_TAG $([[ "$GIT_REF" != "$RELEASE_TAG" ]] && printf "\n%s\n" "GIT_REF:         $GIT_REF")
 263 API_DIFF_TAG:    $API_DIFF_TAG
 264 ASF_USERNAME:    $ASF_USERNAME
 265 GPG_KEY:         $GPG_KEY
 266 GIT_NAME:        $GIT_NAME
 267 GIT_EMAIL:       $GIT_EMAIL
 268 DRY_RUN:         $(is_dry_run && echo "yes" || echo "NO, THIS BUILD WILL BE PUBLISHED!")
 269 ================
 270 EOF
 271
 272   read -r -p "Is this info correct [y/n]? " ANSWER
 273   if [ "$ANSWER" != "y" ]; then
 274     log "Exiting."
 275     exit 1
 276   fi
 277   GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}")
 278
 279   if ! is_dry_run; then
 280     if [ -z "$ASF_PASSWORD" ]; then
 281       stty -echo && printf "ASF_PASSWORD: " && read -r ASF_PASSWORD && printf '\n' && stty echo
 282     fi
 283   else
 284     ASF_PASSWORD="***INVALID***"
 285   fi
 286
 287   export ASF_PASSWORD
 288 }
 289
 290 function is_dry_run {
 291   [[ "$DRY_RUN" = 1 ]]
 292 }
 293
 294 function is_debug {
 295   [[ "${DEBUG}" = 1 ]]
 296 }
 297
 298 function check_get_passwords {
 299   for env in "$@"; do
 300     if [ -z "${!env}" ]; then
 301       log "The environment variable $env is not set. Please enter the password or passphrase."
 302       echo
 303       # shellcheck disable=SC2229
 304       stty -echo && printf "%s : " "$env" && read -r "$env" && printf '\n' && stty echo
 305     fi
 306     # shellcheck disable=SC2163
 307     export "$env"
 308   done
 309 }
 310
 311 function check_needed_vars {
 312   local missing=0
 313   for env in "$@"; do
 314     if [ -z "${!env}" ]; then
 315       log "$env must be set to run this script"
 316       (( missing++ ))
 317     else
 318       # shellcheck disable=SC2163
 319       export "$env"
 320     fi
 321   done
 322   (( missing > 0 )) && exit_with_usage
 323   return 0
 324 }
 325
 326 function init_locale {
 327   local locale_value
 328   OS="$(uname -s)"
 329   case "${OS}" in
 330     Darwin*)    locale_value="en_US.UTF-8";;
 331     Linux*)     locale_value="C.UTF-8";;
 332     *)          error "unknown OS";;
 333   esac
 334   export LC_ALL="$locale_value"
 335   export LANG="$locale_value"
 336 }
 337
 338 # Initializes JAVA_VERSION to the version of the JVM in use.
 339 function init_java {
 340   if [ -z "$JAVA_HOME" ]; then
 341     error "JAVA_HOME is not set."
 342   fi
 343   JAVA_VERSION=$("${JAVA_HOME}"/bin/javac -version 2>&1 | cut -d " " -f 2)
 344   log "java version: $JAVA_VERSION"
 345   export JAVA_VERSION
 346 }
 347
 348 function init_python {
 349   if ! [ -x "$(command -v python2)"  ]; then
 350     error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2'
 351   fi
 352   log "python version: $(python2 --version)"
 353 }
 354
 355 # Set MVN
 356 function init_mvn {
 357   if [ -n "$MAVEN_HOME" ]; then
 358       MVN=("${MAVEN_HOME}/bin/mvn")
 359   elif [ "$(type -P mvn)" ]; then
 360       MVN=(mvn)
 361   else
 362     error "MAVEN_HOME is not set nor is mvn on the current path."
 363   fi
 364   # Add batch mode.
 365   MVN=("${MVN[@]}" -B)
 366   export MVN
 367   echo -n "mvn version: "
 368   "${MVN[@]}" --version
 369   configure_maven
 370 }
 371
 372 function init_yetus {
 373   declare YETUS_VERSION
 374   if [ -z "${YETUS_HOME}" ]; then
 375     error "Missing Apache Yetus."
 376   fi
 377   # Work around yetus bug by asking test-patch for the version instead of rdm.
 378   YETUS_VERSION=$("${YETUS_HOME}/bin/test-patch" --version)
 379   log "Apache Yetus version ${YETUS_VERSION}"
 380 }
 381
 382 function configure_maven {
 383   # Add timestamps to mvn logs.
 384   MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}"
 385   # Suppress gobs of "Download from central:" messages
 386   MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn ${MAVEN_OPTS}"
 387   MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}"
 388   [[ -d "$MAVEN_LOCAL_REPO" ]] || mkdir -p "$MAVEN_LOCAL_REPO"
 389   MAVEN_SETTINGS_FILE="${MAVEN_LOCAL_REPO}/tmp-settings.xml"
 390   MVN=("${MVN[@]}" --settings "${MAVEN_SETTINGS_FILE}")
 391   export MVN MAVEN_OPTS MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO
 392   export ASF_USERNAME ASF_PASSWORD
 393   # reference passwords from env rather than storing in the settings.xml file.
 394   cat <<'EOF' > "$MAVEN_SETTINGS_FILE"
 395 <?xml version="1.0" encoding="UTF-8"?>
 396 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
 397           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 398           xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
 399   <localRepository>/${env.MAVEN_LOCAL_REPO}</localRepository>
 400   <servers>
 401     <server><id>apache.snapshots.https</id><username>${env.ASF_USERNAME}</username>
 402       <password>${env.ASF_PASSWORD}</password></server>
 403     <server><id>apache.releases.https</id><username>${env.ASF_USERNAME}</username>
 404       <password>${env.ASF_PASSWORD}</password></server>
 405   </servers>
 406   <profiles>
 407     <profile>
 408       <activation>
 409         <activeByDefault>true</activeByDefault>
 410       </activation>
 411       <properties>
 412         <gpg.keyname>${env.GPG_KEY}</gpg.keyname>
 413       </properties>
 414     </profile>
 415   </profiles>
 416 </settings>
 417 EOF
 418 }
 419
 420 # clone of the repo, deleting anything that exists in the working directory named after the project.
 421 # optionally with auth details for pushing.
 422 function git_clone_overwrite {
 423   local asf_repo
 424   if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
 425     error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
 426   fi
 427   rm -rf "${PROJECT}"
 428
 429   if [[ -z "${GIT_REPO}" ]]; then
 430     asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
 431     log "Clone will be of the gitbox repo for ${PROJECT}."
 432     if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
 433       # Ugly!
 434       encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')")
 435       encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')")
 436       GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
 437     else
 438       GIT_REPO="https://${asf_repo}"
 439     fi
 440   else
 441     log "Clone will be of provided git repo."
 442   fi
 443   # N.B. we use the shared flag because the clone is short lived and if a local repo repo was
 444   #      given this will let us refer to objects there directly instead of hardlinks or copying.
 445   #      The option is silently ignored for non-local repositories. see the note on git help clone
 446   #      for the --shared option for details.
 447   git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
 448   # If this was a host local git repo then add in an alternates and remote that will
 449   # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
 450   # for more info see 'git help remote' and 'git help repository-layout'.
 451   if [ -n "$HOST_GIT_REPO" ]; then
 452     echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
 453     (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
 454   fi
 455 }
 456
 457 function start_step {
 458   local name=$1
 459   if [ -z "${name}" ]; then
 460     name="${FUNCNAME[1]}"
 461   fi
 462   log "${name} start" >&2
 463   get_ctime
 464 }
 465
 466 function stop_step {
 467   local name=$2
 468   local start_time=$1
 469   local stop_time
 470   if [ -z "${name}" ]; then
 471     name="${FUNCNAME[1]}"
 472   fi
 473   stop_time="$(get_ctime)"
 474   log "${name} stop ($((stop_time - start_time)) seconds)"
 475 }
 476
 477 # Writes report into cwd!
 478 # TODO should have option for maintenance release that include LimitedPrivate in report
 479 function generate_api_report {
 480   local project="$1"
 481   local previous_tag="$2"
 482   local release_tag="$3"
 483   local previous_version
 484   local timing_token
 485   timing_token="$(start_step)"
 486   # Generate api report.
 487   # Filter out some jar types. Filters are tricky. Python regex on
 488   # file basename. Exclude the saved-aside original jars... they are
 489   # not included in resulting artifact. Also, do not include the
 490   # hbase-shaded-testing-util.*  jars. This jar is unzip'able on mac
 491   # os x as is because has it a META_INF/LICENSE file and then a
 492   # META_INF/license directory for the included jar's licenses;
 493   # it fails to unjar on mac os x which this tool does making its checks
 494   # (Its exclusion should be fine; it is just an aggregate of other jars).
 495   "${project}"/dev-support/checkcompatibility.py --annotation \
 496     org.apache.yetus.audience.InterfaceAudience.Public  \
 497     -e "original-hbase.*.jar" \
 498     -e "hbase-shaded-testing-util.*.jar" \
 499     "$previous_tag" "$release_tag"
 500   previous_version="$(echo "${previous_tag}" | sed -e 's/rel\///')"
 501   cp "${project}/target/compat-check/report.html" "./api_compare_${previous_version}_to_${release_tag}.html"
 502   stop_step "${timing_token}"
 503 }
 504
 505 # Look up the Jira name associated with project.
 506 # Returns result on stdout.
 507 # Currently all the 'hbase-*' projects share the same HBASE jira name.  This works because,
 508 # by convention, the HBASE jira "Fix Version" field values have the sub-project name pre-pended,
 509 # as in "hbase-operator-tools-1.0.0".
 510 # TODO: For non-hbase-related projects, enhance this to use Jira API query instead of text lookup.
 511 function get_jira_name {
 512   local project="$1"
 513   local jira_name
 514   case "${project}" in
 515     hbase*) jira_name="HBASE";;
 516     *)      jira_name="";;
 517   esac
 518   if [[ -z "$jira_name" ]]; then
 519     error "Sorry, can't determine the Jira name for project $project"
 520   fi
 521   echo "$jira_name"
 522 }
 523
 524 # Update the CHANGES.md
 525 # DOES NOT DO COMMITS! Caller should do that.
 526 # requires yetus to have a defined home already.
 527 # yetus requires python2 to be on the path.
 528 function update_releasenotes {
 529   local project_dir="$1"
 530   local jira_fix_version="$2"
 531   local jira_project
 532   local timing_token
 533   timing_token="$(start_step)"
 534   changelog="CHANGELOG.${jira_fix_version}.md"
 535   releasenotes="RELEASENOTES.${jira_fix_version}.md"
 536   if [ -f ${changelog} ]; then
 537     rm ${changelog}
 538   fi
 539   if [ -f ${releasenotes} ]; then
 540     rm ${releasenotes}
 541   fi
 542   jira_project="$(get_jira_name "$(basename "$project_dir")")"
 543   "${YETUS_HOME}/bin/releasedocmaker" -p "${jira_project}" --fileversions -v "${jira_fix_version}" \
 544       -l --sortorder=newer --skip-credits || true
 545   # First clear out the changes written by previous RCs.
 546   if [ -f "${project_dir}/CHANGES.md" ]; then
 547     sed -i -e \
 548         "/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \
 549         "${project_dir}/CHANGES.md" || true
 550   fi
 551   if [ -f "${project_dir}/RELEASENOTES.md" ]; then
 552     sed -i -e \
 553         "/^# ${jira_project}  ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project}  ${jira_fix_version} Release Notes/d;}" \
 554         "${project_dir}/RELEASENOTES.md" || true
 555   fi
 556
 557   # Yetus will not generate CHANGES if no JIRAs fixed against the release version
 558   # (Could happen if a release were bungled such that we had to make a new one
 559   # without changes)
 560   if [ ! -f "${changelog}" ]; then
 561     echo -e "## Release ${jira_fix_version} - Unreleased (as of `date`)\nNo changes\n" > "${changelog}"
 562   fi
 563   if [ ! -f "${releasenotes}" ]; then
 564     echo -e "# hbase ${jira_fix_version} Release Notes\nNo changes\n" > "${releasenotes}"
 565   fi
 566
 567   # The releasedocmaker call above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md.
 568   if [ -f "${project_dir}/CHANGES.md" ]; then
 569     # To insert into project's CHANGES.md...need to cut the top off the
 570     # CHANGELOG.X.X.X.md file removing license and first line and then
 571     # insert it after the license comment closing where we have a
 572     # DO NOT REMOVE marker text!
 573     sed -i -e '/## Release/,$!d' "${changelog}"
 574     sed -i -e '2,${/^# HBASE Changelog/d;}' "${project_dir}/CHANGES.md"
 575     sed -i -e "/DO NOT REMOVE/r ${changelog}" "${project_dir}/CHANGES.md"
 576   else
 577     mv "${changelog}" "${project_dir}/CHANGES.md"
 578   fi
 579   if [ -f "${project_dir}/RELEASENOTES.md" ]; then
 580     # Similar for RELEASENOTES but slightly different.
 581     sed -i -e '/Release Notes/,$!d' "${releasenotes}"
 582     sed -i -e '2,${/^# RELEASENOTES/d;}' "${project_dir}/RELEASENOTES.md"
 583     sed -i -e "/DO NOT REMOVE/r ${releasenotes}" "${project_dir}/RELEASENOTES.md"
 584   else
 585     mv "${releasenotes}" "${project_dir}/RELEASENOTES.md"
 586   fi
 587   stop_step "${timing_token}"
 588 }
 589
 590 # Make src release.
 591 # Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
 592 # -- and then the version string. Expects to find checkout adjacent to this script
 593 # named for 'project', the first arg passed.
 594 # Expects the following three defines in the environment:
 595 # - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
 596 # - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
 597 # For example:
 598 # $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
 599 make_src_release() {
 600   # Tar up the src and sign and hash it.
 601   local project="${1}"
 602   local version="${2}"
 603   local base_name="${project}-${version}"
 604   local timing_token
 605   timing_token="$(start_step)"
 606   rm -rf "${base_name}"-src*
 607   tgz="${base_name}-src.tar.gz"
 608   cd "${project}" || exit
 609   git clean -d -f -x
 610   git archive --format=tar.gz --output="../${tgz}" --prefix="${base_name}/" "${GIT_REF:-master}"
 611   cd .. || exit
 612   $GPG "${GPG_ARGS[@]}" --armor --output "${tgz}.asc" --detach-sig "${tgz}"
 613   $GPG "${GPG_ARGS[@]}" --print-md SHA512 "${tgz}" > "${tgz}.sha512"
 614   stop_step "${timing_token}"
 615 }
 616
 617 # Make binary release.
 618 # Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
 619 # -- and then the version string. Expects to find checkout adjacent to this script
 620 # named for 'project', the first arg passed.
 621 # Expects the following three defines in the environment:
 622 # - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
 623 # - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
 624 # - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE".
 625 # For example:
 626 # $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
 627 make_binary_release() {
 628   local project="${1}"
 629   local version="${2}"
 630   local base_name="${project}-${version}"
 631   local timing_token
 632   timing_token="$(start_step)"
 633   rm -rf "${base_name}"-bin*
 634   cd "$project" || exit
 635
 636   git clean -d -f -x
 637   # Three invocations of maven. This seems to work. One to
 638   # populate the repo, another to build the site, and then
 639   # a third to assemble the binary artifact. Trying to do
 640   # all in the one invocation fails; a problem in our
 641   # assembly spec to in maven. TODO. Meantime, three invocations.
 642   "${MVN[@]}" clean install -DskipTests
 643   "${MVN[@]}" site -DskipTests
 644   kick_gpg_agent
 645   "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}"
 646
 647   # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty.
 648   local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}"
 649   if ls "${f_bin_prefix}"*-bin.tar.gz &>/dev/null; then
 650     cp "${f_bin_prefix}"*-bin.tar.gz ..
 651     cd .. || exit
 652     for i in "${base_name}"*-bin.tar.gz; do
 653       "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}"
 654       "${GPG}" "${GPG_ARGS[@]}" --print-md SHA512 "${i}" > "${i}.sha512"
 655     done
 656   else
 657     cd .. || exit
 658     log "No ${f_bin_prefix}*-bin.tar.gz product; expected?"
 659   fi
 660
 661   stop_step "${timing_token}"
 662 }
 663
 664 # "Wake up" the gpg agent so it responds properly to maven-gpg-plugin, and doesn't cause timeout.
 665 # Specifically this is done between invocation of 'mvn site' and 'mvn assembly:single', because
 666 # the 'site' build takes long enough that the gpg-agent does become unresponsive and the following
 667 # 'assembly' build (where gpg signing occurs) experiences timeout, without this "kick".
 668 function kick_gpg_agent {
 669   # All that's needed is to run gpg on a random file
 670   # TODO could we just call gpg-connect-agent /bye
 671   local i
 672   i="$(mktemp)"
 673   echo "This is a test file" > "$i"
 674   "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}"
 675   rm "$i" "$i.asc"
 676 }
 677
 678 # Do maven command to set version into local pom
 679 function maven_set_version { #input: <version_to_set>
 680   local this_version="$1"
 681   log "${MVN[@]}" versions:set -DnewVersion="$this_version"
 682   "${MVN[@]}" versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs
 683 }
 684
 685 # Do maven command to read version from local pom
 686 function maven_get_version {
 687   # shellcheck disable=SC2016
 688   "${MVN[@]}" -q -N -Dexec.executable="echo" -Dexec.args='${project.version}' exec:exec
 689 }
 690
 691 # Do maven deploy to snapshot or release artifact repository, with checks.
 692 function maven_deploy { #inputs: <snapshot|release> <log_file_path>
 693   local timing_token
 694   # Invoke with cwd=$PROJECT
 695   local deploy_type="$1"
 696   local mvn_log_file="$2" #secondary log file used later to extract staged_repo_id
 697   if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then
 698     error "unrecognized deploy type, must be 'snapshot'|'release'"
 699   fi
 700   if [[ -z "$mvn_log_file" ]] || ! touch "$mvn_log_file"; then
 701     error "must provide writable maven log output filepath"
 702   fi
 703   timing_token=$(start_step)
 704   # shellcheck disable=SC2153
 705   if [[ "$deploy_type" == "snapshot" ]] && ! [[ "$RELEASE_VERSION" =~ -SNAPSHOT$ ]]; then
 706     error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$RELEASE_VERSION'"
 707   elif [[ "$deploy_type" == "release" ]] && [[ "$RELEASE_VERSION" =~ SNAPSHOT ]]; then
 708     error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$RELEASE_VERSION'"
 709   fi
 710   # Publish ${PROJECT} to Maven repo
 711   # shellcheck disable=SC2154
 712   log "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)"
 713   log "Publish version is $RELEASE_VERSION"
 714   # Coerce the requested version
 715   maven_set_version "$RELEASE_VERSION"
 716   # Prepare for signing
 717   kick_gpg_agent
 718   declare -a mvn_goals=(clean)
 719   if ! is_dry_run; then
 720     mvn_goals=("${mvn_goals[@]}" deploy)
 721   fi
 722   log "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" "${mvn_goals[@]}"
 723   log "Logging to ${mvn_log_file}.  This will take a while..."
 724   rm -f "$mvn_log_file"
 725   # The tortuous redirect in the next command allows mvn's stdout and stderr to go to mvn_log_file,
 726   # while also sending stderr back to the caller.
 727   # shellcheck disable=SC2094
 728   if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \
 729       "${mvn_goals[@]}" 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then
 730     error "Deploy build failed, for details see log at '$mvn_log_file'."
 731   fi
 732   log "BUILD SUCCESS."
 733   stop_step "${timing_token}"
 734   return 0
 735 }
 736
 737 # guess the host os
 738 # * DARWIN
 739 # * LINUX
 740 function get_host_os() {
 741   uname -s | tr '[:lower:]' '[:upper:]'
 742 }