dev-support/jenkins-scripts/cache-apache-project-artifact.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e
  20 function usage {
  21   echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
  22   echo ""
  23   echo "    --force                       for a redownload even if /path/to/download/file.tar.gz exists."
  24   echo "    --working-dir /path/to/use    Path for writing tempfiles. must exist."
  25   echo "                                  defaults to making a directory via mktemp that we clean."
  26   echo "    --keys url://to/project/KEYS  where to get KEYS. needed to check signature on download."
  27   echo ""
  28   exit 1
  29 }
  30 # if no args specified, show usage
  31 if [ $# -lt 2 ]; then
  32   usage
  33 fi
  34
  35
  36 # Get arguments
  37 declare done_if_cached="true"
  38 declare working_dir
  39 declare cleanup="true"
  40 declare keys
  41 while [ $# -gt 0 ]
  42 do
  43   case "$1" in
  44     --force) shift; done_if_cached="false";;
  45     --working-dir) shift; working_dir=$1; cleanup="false"; shift;;
  46     --keys) shift; keys=$1; shift;;
  47     --) shift; break;;
  48     -*) usage ;;
  49     *)  break;;  # terminate while loop
  50   esac
  51 done
  52
  53 # should still have required args
  54 if [ $# -lt 2 ]; then
  55   usage
  56 fi
  57
  58 target="$1"
  59 artifact="$2"
  60
  61 if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
  62   echo "Reusing existing download of '${artifact}'."
  63   exit 0
  64 fi
  65
  66 if [ -z "${working_dir}" ]; then
  67   if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then
  68     echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
  69     exit 1
  70   fi
  71 else
  72   # absolutes please
  73   working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
  74   if [ ! -d "${working_dir}" ]; then
  75     echo "passed working directory '${working_dir}' must already exist." >&2
  76     exit 1
  77   fi
  78 fi
  79
  80 function cleanup {
  81   if [ "true" = "${cleanup}" ]; then
  82     echo "cleaning up temp space."
  83     rm -rf "${working_dir}"
  84   fi
  85 }
  86 trap cleanup EXIT SIGQUIT
  87
  88 echo "New download of '${artifact}'"
  89
  90 # N.B. this comes first so that if gpg falls over we skip the expensive download.
  91 if [ -n "${keys}" ]; then
  92   if [ ! -d "${working_dir}/.gpg" ]; then
  93     rm -rf "${working_dir}/.gpg"
  94     mkdir -p "${working_dir}/.gpg"
  95     chmod -R 700 "${working_dir}/.gpg"
  96   fi
  97
  98   echo "installing project KEYS"
  99   curl -L --fail -o "${working_dir}/KEYS" "${keys}"
 100   if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then
 101     echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2
 102     echo "    gpg: can't connect to the agent: File name too long" >&2
 103     # we mean to give them the command to run, not to run it.
 104     #shellcheck disable=SC2016
 105     echo 'then you prolly need to create /var/run/user/$(id -u)' >&2
 106     echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2
 107     exit 2
 108   fi
 109
 110   echo "downloading signature"
 111   curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc"
 112 fi
 113
 114 echo "downloading artifact"
 115 if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then
 116   echo "Artifact wasn't in mirror system. falling back to archive.a.o."
 117   curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}"
 118 fi
 119
 120 if [ -n "${keys}" ]; then
 121   echo "verifying artifact signature"
 122   gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc"
 123   echo "signature good."
 124 fi
 125
 126 echo "moving artifact into place at '${target}'"
 127 # ensure we're on the same filesystem
 128 mv "${working_dir}/artifact" "${target}.copying"
 129 # attempt atomic move
 130 mv "${target}.copying" "${target}"
 131 echo "all done!"