dev-support/jenkins-scripts/generate-hbase-website.sh

   1 #!/bin/bash
   2 #
   3 #/**
   4 # * Licensed to the Apache Software Foundation (ASF) under one
   5 # * or more contributor license agreements.  See the NOTICE file
   6 # * distributed with this work for additional information
   7 # * regarding copyright ownership.  The ASF licenses this file
   8 # * to you under the Apache License, Version 2.0 (the
   9 # * "License"); you may not use this file except in compliance
  10 # * with the License.  You may obtain a copy of the License at
  11 # *
  12 # *     http://www.apache.org/licenses/LICENSE-2.0
  13 # *
  14 # * Unless required by applicable law or agreed to in writing, software
  15 # * distributed under the License is distributed on an "AS IS" BASIS,
  16 # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 # * See the License for the specific language governing permissions and
  18 # * limitations under the License.
  19 # */
  20
  21 # This script is meant to run as part of a Jenkins job such as
  22 # https://builds.apache.org/job/hbase_generate_website/
  23 #
  24 # It needs to be built on a Jenkins server with the label git-websites
  25 #
  26 # Allows specifying options for working directory, maven repo, and publishing to git
  27 # run with --help for usage.
  28 #
  29 # If there is a build error, the Jenkins job is configured to send an email
  30
  31 declare CURRENT_HBASE_COMMIT
  32 declare PUSHED
  33 declare FILE
  34 declare WEBSITE_COMMIT_MSG
  35 declare -a FILES_TO_REMOVE
  36
  37 set -e
  38 function usage {
  39   echo "Usage: ${0} [options] /path/to/hbase/checkout"
  40   echo ""
  41   echo "    --working-dir /path/to/use  Path for writing logs and a local checkout of hbase-site repo."
  42   echo "                                if given must exist."
  43   echo "                                defaults to making a directory via mktemp."
  44   echo "    --local-repo /path/for/maven/.m2  Path for putting local maven repo."
  45   echo "                                if given must exist."
  46   echo "                                defaults to making a clean directory in --working-dir."
  47   echo "    --publish                   if given, will attempt to push results back to the hbase-site repo."
  48   echo "    --help                      show this usage message."
  49   exit 1
  50 }
  51 # if no args specified, show usage
  52 if [ $# -lt 1 ]; then
  53   usage
  54 fi
  55
  56 # Get arguments
  57 declare component_dir
  58 declare working_dir
  59 declare local_repo
  60 declare publish
  61 while [ $# -gt 0 ]
  62 do
  63   case "$1" in
  64     --working-dir) shift; working_dir=$1; shift;;
  65     --local-repo) shift; local_repo=$1; shift;;
  66     --publish) shift; publish="true";;
  67     --) shift; break;;
  68     -*) usage ;;
  69     *)  break;;  # terminate while loop
  70   esac
  71 done
  72
  73 # should still have where component checkout is.
  74 if [ $# -lt 1 ]; then
  75   usage
  76 fi
  77 component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
  78
  79 if [ -z "${working_dir}" ]; then
  80   echo "[DEBUG] defaulting to creating a directory via mktemp"
  81   if ! working_dir="$(mktemp -d -t hbase-generate-website)" ; then
  82     echo "Failed to create temporary working directory. Please specify via --working-dir"
  83     exit 1
  84   fi
  85 else
  86   # absolutes please
  87   working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
  88   if [ ! -d "${working_dir}" ]; then
  89     echo "passed working directory '${working_dir}' must already exist."
  90     exit 1
  91   fi
  92 fi
  93
  94 echo "You'll find logs and temp files in ${working_dir}"
  95
  96 if [ -z "${local_repo}" ]; then
  97   echo "[DEBUG] defaulting to creating a local repo within '${working_dir}'"
  98   local_repo="${working_dir}/.m2/repo"
  99   # Nuke the local maven repo each time, to start with a known environment
 100   rm -Rf "${local_repo}"
 101   mkdir -p "${local_repo}"
 102 else
 103   # absolutes please
 104   local_repo="$(cd "$(dirname "${local_repo}")"; pwd)/$(basename "${local_repo}")"
 105   if [ ! -d "${local_repo}" ]; then
 106     echo "passed directory for storing the maven repo '${local_repo}' must already exist."
 107     exit 1
 108   fi
 109 fi
 110
 111 # Set up the environment
 112 if [ -z "${JAVA_HOME}" ]; then
 113   JAVA_HOME="${JDK_1_8_LATEST__HOME}"
 114   export JAVA_HOME
 115   export PATH="${JAVA_HOME}/bin:${PATH}"
 116 fi
 117 if [ -z "${MAVEN_HOME}" ]; then
 118   MAVEN_HOME="${MAVEN_3_3_3_HOME}"
 119   export MAVEN_HOME
 120   export PATH="${MAVEN_HOME}/bin:${PATH}"
 121 fi
 122 export MAVEN_OPTS="${MAVEN_OPTS} -Dmaven.repo.local=${local_repo}"
 123
 124 # Verify the Maven version
 125 mvn -version
 126 # Verify the git version
 127 git --version
 128
 129 cd "${working_dir}"
 130
 131 # Clean any leftover files in case we are reusing the workspace
 132 rm -Rf -- *.patch *.patch.zip target *.txt hbase-site
 133
 134 # Save and print the SHA we are building
 135 CURRENT_HBASE_COMMIT="$(cd "${component_dir}" && git show-ref --hash --dereference --verify refs/remotes/origin/HEAD)"
 136 # Fail if it's empty
 137 if [ -z "${CURRENT_HBASE_COMMIT}" ]; then
 138   echo "Got back a blank answer for the current HEAD on the remote hbase repository. failing."
 139   exit 1
 140 fi
 141 echo "Current HBase commit: $CURRENT_HBASE_COMMIT"
 142
 143 # Clone the hbase-site repo manually so it doesn't trigger spurious
 144 # commits in Jenkins.
 145 git clone --depth 1 --branch asf-site https://gitbox.apache.org/repos/asf/hbase-site.git
 146
 147 # Figure out if the commit of the hbase repo has already been built and bail if so.
 148 declare -i PUSHED
 149 PUSHED=$(cd hbase-site && git rev-list --grep "${CURRENT_HBASE_COMMIT}" --fixed-strings --count HEAD)
 150 echo "[DEBUG] hash was found in $PUSHED commits for hbase-site repository."
 151
 152 if [ "${PUSHED}" -ne 0 ]; then
 153   echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building."
 154   exit 0
 155 else
 156   echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet."
 157 fi
 158
 159 # Go to the hbase directory so we can build the site
 160 cd "${component_dir}"
 161
 162 # This will only be set for builds that are triggered by SCM change, not manual builds
 163 if [ -n "$CHANGE_ID" ]; then
 164   echo -n " ($CHANGE_ID - $CHANGE_TITLE)"
 165 fi
 166
 167 # Build and install HBase, then build the site
 168 echo "Building HBase"
 169 # TODO we have to do a local install first because for whatever reason, the maven-javadoc-plugin's
 170 # forked compile phase requires that test-scoped dependencies be available, which
 171 # doesn't work since we will not have done a test-compile phase (MJAVADOC-490). the first place this
 172 # breaks for me is hbase-server trying to find hbase-http:test and hbase-zookeeper:test.
 173 # But! some sunshine: because we're doing a full install before running site, we can skip all the
 174 # compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
 175 if mvn \
 176     --batch-mode \
 177     -Psite-install-step \
 178     --errors \
 179     --log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
 180     clean install \
 181   && mvn site \
 182     --batch-mode \
 183     -Dscala.skip=true \
 184     -Psite-build-step \
 185     --errors \
 186     --log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
 187   echo "Successfully built site."
 188 else
 189   status=$?
 190   echo "Maven commands to build the site failed. check logs for details ${working_dir}/hbase-*-log-*.txt"
 191   exit $status
 192 fi
 193
 194 # Stage the site
 195 echo "Staging HBase site"
 196 mvn \
 197   --batch-mode \
 198   --errors \
 199   --log-file="${working_dir}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \
 200   site:stage
 201 status=$?
 202 if [ $status -ne 0 ] || [ ! -d target/staging ]; then
 203   echo "Failure: mvn site:stage"
 204   exit $status
 205 fi
 206
 207 # Get ready to update the hbase-site repo with the new artifacts
 208 cd "${working_dir}/hbase-site"
 209
 210 #Remove previously-generated files
 211 FILES_TO_REMOVE=("hbase-*"
 212                  "apidocs"
 213                  "devapidocs"
 214                  "testapidocs"
 215                  "testdevapidocs"
 216                  "xref"
 217                  "xref-test"
 218                  "*book*"
 219                  "*.html"
 220                  "*.pdf*"
 221                  "css"
 222                  "js"
 223                  "images")
 224
 225 for FILE in "${FILES_TO_REMOVE[@]}"; do
 226   if [ -e "${FILE}" ]; then
 227     echo "Removing hbase-site/$FILE"
 228     rm -Rf "${FILE}"
 229   fi
 230 done
 231
 232 # Copy in the newly-built artifacts
 233 # TODO what do we do when the site build wants to remove something? Can't rsync because e.g. release-specific docs.
 234 cp -pPR "${component_dir}"/target/staging/* .
 235
 236 # If the index.html is missing, bail because this is serious
 237 if [ ! -f index.html ]; then
 238   echo "The index.html is missing. Aborting."
 239   exit 1
 240 fi
 241
 242 echo "Adding all the files we know about"
 243 git add .
 244 # Create the commit message and commit the changes
 245 WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT."
 246 echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG"
 247 git commit -m "${WEBSITE_COMMIT_MSG}" -a
 248 # Dump a little report
 249 echo "This commit changed these files (excluding Modified files):"
 250 git diff --name-status --diff-filter=ADCRTXUB origin/asf-site | tee "${working_dir}/hbase-file-diff-summary-${CURRENT_HBASE_COMMIT}.txt"
 251 # Create a patch, which Jenkins can save as an artifact and can be examined for debugging
 252 git format-patch --stdout origin/asf-site > "${working_dir}/${CURRENT_HBASE_COMMIT}.patch"
 253 if [ ! -s "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" ]; then
 254   echo "Something went wrong when creating the patch of our updated site."
 255   exit 1
 256 fi
 257 echo "Change set saved to patch ${working_dir}/${CURRENT_HBASE_COMMIT}.patch"
 258
 259 if [ -n "${publish}" ]; then
 260   echo "Publishing changes to remote repo..."
 261   if git push origin asf-site; then
 262     echo "changes pushed."
 263   else
 264     echo "Failed to push to asf-site. Website not updated."
 265     exit 1
 266   fi
 267   echo "Sending empty commit to work around INFRA-10751."
 268   git commit --allow-empty -m "INFRA-10751 Empty commit"
 269   # Push the empty commit
 270   if git push origin asf-site; then
 271     echo "empty commit pushed."
 272   else
 273     echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)"
 274     exit 1
 275   fi
 276   echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes."
 277 fi
 278
 279 # Zip up the patch so Jenkins can save it
 280 cd "${working_dir}"
 281 zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch"