dev-support/jenkins-scripts/cache-apache-project-artifact.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e
  20 function usage {
  21   echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
  22   echo ""
  23   echo "    --force                       for a redownload even if /path/to/download/file.tar.gz exists."
  24   echo "    --verify-tar-gz               Only use a cached file if it can be parsed as a gzipped tarball."
  25   echo "    --working-dir /path/to/use    Path for writing tempfiles. must exist."
  26   echo "                                  defaults to making a directory via mktemp that we clean."
  27   echo "    --keys url://to/project/KEYS  where to get KEYS. needed to check signature on download."
  28   echo ""
  29   exit 1
  30 }
  31 # if no args specified, show usage
  32 if [ $# -lt 2 ]; then
  33   usage
  34 fi
  35
  36
  37 # Get arguments
  38 declare done_if_cached="true"
  39 declare verify_tar_gz="false"
  40 declare working_dir
  41 declare cleanup="true"
  42 declare keys
  43 while [ $# -gt 0 ]
  44 do
  45   case "$1" in
  46     --force) shift; done_if_cached="false";;
  47     --verify-tar-gz) shift; verify_tar_gz="true";;
  48     --working-dir) shift; working_dir=$1; cleanup="false"; shift;;
  49     --keys) shift; keys=$1; shift;;
  50     --) shift; break;;
  51     -*) usage ;;
  52     *)  break;;  # terminate while loop
  53   esac
  54 done
  55
  56 # should still have required args
  57 if [ $# -lt 2 ]; then
  58   usage
  59 fi
  60
  61 target="$1"
  62 artifact="$2"
  63
  64 if [ -f "${target}" ] && [ -s "${target}" ] && [ -r "${target}" ] && [ "true" = "${done_if_cached}" ]; then
  65   if [ "false" = "${verify_tar_gz}" ]; then
  66     echo "Reusing existing download of '${artifact}'."
  67     exit 0
  68   fi
  69   if ! tar tzf "${target}" > /dev/null 2>&1; then
  70     echo "Cached artifact is not a well formed gzipped tarball; clearing the cached file at '${target}'."
  71     rm -rf "${target}"
  72   else
  73     echo "Reusing existing download of '${artifact}', which is a well formed gzipped tarball."
  74     exit 0
  75   fi
  76 fi
  77
  78 if [ -z "${working_dir}" ]; then
  79   if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then
  80     echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
  81     exit 1
  82   fi
  83 else
  84   # absolutes please
  85   working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
  86   if [ ! -d "${working_dir}" ]; then
  87     echo "passed working directory '${working_dir}' must already exist." >&2
  88     exit 1
  89   fi
  90 fi
  91
  92 function cleanup {
  93   if [ -n "${keys}" ]; then
  94     echo "Stopping gpg agent daemon"
  95     gpgconf --homedir "${working_dir}/.gpg" --kill gpg-agent
  96     echo "Stopped gpg agent daemon"
  97   fi
  98
  99   if [ "true" = "${cleanup}" ]; then
 100     echo "cleaning up temp space."
 101     rm -rf "${working_dir}"
 102   fi
 103 }
 104 trap cleanup EXIT SIGQUIT
 105
 106 echo "New download of '${artifact}'"
 107
 108 # N.B. this comes first so that if gpg falls over we skip the expensive download.
 109 if [ -n "${keys}" ]; then
 110   if [ ! -d "${working_dir}/.gpg" ]; then
 111     rm -rf "${working_dir}/.gpg"
 112     mkdir -p "${working_dir}/.gpg"
 113     chmod -R 700 "${working_dir}/.gpg"
 114   fi
 115
 116   echo "installing project KEYS"
 117   curl -L --fail -o "${working_dir}/KEYS" "${keys}"
 118   if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then
 119     echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2
 120     echo "    gpg: can't connect to the agent: File name too long" >&2
 121     # we mean to give them the command to run, not to run it.
 122     #shellcheck disable=SC2016
 123     echo 'then you prolly need to create /var/run/user/$(id -u)' >&2
 124     echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2
 125     exit 2
 126   fi
 127
 128   echo "downloading signature"
 129   curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc"
 130 fi
 131
 132 echo "downloading artifact"
 133 if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then
 134   echo "Artifact wasn't in mirror system. falling back to archive.a.o."
 135   curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}"
 136 fi
 137
 138 if [ -n "${keys}" ]; then
 139   echo "verifying artifact signature"
 140   gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc"
 141   echo "signature good."
 142 fi
 143
 144 echo "moving artifact into place at '${target}'"
 145 # ensure we're on the same filesystem
 146 mv "${working_dir}/artifact" "${target}.copying"
 147 # attempt atomic move
 148 mv "${target}.copying" "${target}"
 149 echo "all done!"