support/download/dl-wrapper

   1 #!/usr/bin/env bash
   2
   3 # This script is a wrapper to the other download backends.
   4 # Its role is to ensure atomicity when saving downloaded files
   5 # back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
   6 # failed downloads.
   7 #
   8 # Call it with -h to see some help.
   9
  10 # To avoid cluttering BR2_DL_DIR, we download to a trashable
  11 # location, namely in $(BUILD_DIR).
  12 # Then, we move the downloaded file to a temporary file in the
  13 # same directory as the final output file.
  14 # This allows us to finally atomically rename it to its final
  15 # name.
  16 # If anything goes wrong, we just remove all the temporaries
  17 # created so far.
  18
  19 # We want to catch any unexpected failure, and exit immediately.
  20 set -e
  21
  22 main() {
  23     local OPT OPTARG
  24     local backend output hfile quiet
  25
  26     # Parse our options; anything after '--' is for the backend
  27     while getopts :hb:o:H:q OPT; do
  28         case "${OPT}" in
  29         h)  help; exit 0;;
  30         b)  backend="${OPTARG}";;
  31         o)  output="${OPTARG}";;
  32         H)  hfile="${OPTARG}";;
  33         q)  quiet="-q";;
  34         :)  error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
  35         \?) error "unknown option '%s'\n" "${OPTARG}";;
  36         esac
  37     done
  38     # Forget our options, and keep only those for the backend
  39     shift $((OPTIND-1))
  40
  41     if [ -z "${backend}" ]; then
  42         error "no backend specified, use -b\n"
  43     fi
  44     if [ -z "${output}" ]; then
  45         error "no output specified, use -o\n"
  46     fi
  47
  48     # If the output file already exists and:
  49     # - there's no .hash file: do not download it again and exit promptly
  50     # - matches all its hashes: do not download it again and exit promptly
  51     # - fails at least one of its hashes: force a re-download
  52     # - there's no hash (but a .hash file): consider it a hard error
  53     if [ -e "${output}" ]; then
  54         if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
  55             exit 0
  56         elif [ ${?} -ne 2 ]; then
  57             # Do not remove the file, otherwise it might get re-downloaded
  58             # from a later location (i.e. primary -> upstream -> mirror).
  59             # Do not print a message, check-hash already did.
  60             exit 1
  61         fi
  62         rm -f "${output}"
  63         warn "Re-downloading '%s'...\n" "${output##*/}"
  64     fi
  65
  66     # tmpd is a temporary directory in which backends may store intermediate
  67     # by-products of the download.
  68     # tmpf is the file in which the backends should put the downloaded content.
  69     # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
  70     # $(BR2_DL_DIR)
  71     # We let the backends create tmpf, so they are able to set whatever
  72     # permission bits they want (although we're only really interested in
  73     # the executable bit.)
  74     tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
  75     tmpf="${tmpd}/output"
  76
  77     # Helpers expect to run in a directory that is *really* trashable, so
  78     # they are free to create whatever files and/or sub-dirs they might need.
  79     # Doing the 'cd' here rather than in all backends is easier.
  80     cd "${tmpd}"
  81
  82     # If the backend fails, we can just remove the temporary directory to
  83     # remove all the cruft it may have left behind. Then we just exit in
  84     # error too.
  85     if ! "${OLDPWD}/support/download/${backend}" ${quiet} "${tmpf}" "${@}"; then
  86         rm -rf "${tmpd}"
  87         exit 1
  88     fi
  89
  90     # cd back to free the temp-dir, so we can remove it later
  91     cd "${OLDPWD}"
  92
  93     # Check if the downloaded file is sane, and matches the stored hashes
  94     # for that file
  95     if ! support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
  96         rm -rf "${tmpd}"
  97         exit 1
  98     fi
  99
 100     # tmp_output is in the same directory as the final output, so we can
 101     # later move it atomically.
 102     tmp_output="$(mktemp "${output}.XXXXXX")"
 103
 104     # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
 105     # to users other than the one doing the download (and root, of course).
 106     # This can be problematic when a shared BR2_DL_DIR is used by different
 107     # users (e.g. on a build server), where all users may write to the shared
 108     # location, since other users would not be allowed to read the files
 109     # another user downloaded.
 110     # So, we restore the 'go' access rights to a more sensible value, while
 111     # still abiding by the current user's umask. We must do that before the
 112     # final 'mv', so just do it now.
 113     # Some backends (cp and scp) may create executable files, so we need to
 114     # carry the executable bit if needed.
 115     [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
 116     new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
 117     chmod ${new_mode} "${tmp_output}"
 118
 119     # We must *not* unlink tmp_output, otherwise there is a small window
 120     # during which another download process may create the same tmp_output
 121     # name (very, very unlikely; but not impossible.)
 122     # Using 'cp' is not reliable, since 'cp' may unlink the destination file
 123     # if it is unable to open it with O_WRONLY|O_TRUNC; see:
 124     #   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
 125     # Since the destination filesystem can be anything, it might not support
 126     # O_TRUNC, so 'cp' would unlink it first.
 127     # Use 'cat' and append-redirection '>>' to save to the final location,
 128     # since that is the only way we can be 100% sure of the behaviour.
 129     if ! cat "${tmpf}" >>"${tmp_output}"; then
 130         rm -rf "${tmpd}" "${tmp_output}"
 131         exit 1
 132     fi
 133     rm -rf "${tmpd}"
 134
 135     # tmp_output and output are on the same filesystem, so POSIX guarantees
 136     # that 'mv' is atomic, because it then uses rename() that POSIX mandates
 137     # to be atomic, see:
 138     #   http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
 139     if ! mv -f "${tmp_output}" "${output}"; then
 140         rm -f "${tmp_output}"
 141         exit 1
 142     fi
 143 }
 144
 145 help() {
 146     cat <<_EOF_
 147 NAME
 148     ${my_name} - download wrapper for Buildroot
 149
 150 SYNOPSIS
 151     ${my_name} [OPTION]... -- [BACKEND OPTION]...
 152
 153 DESCRIPTION
 154     Wrapper script around different download mechanisms. Ensures that
 155     concurrent downloads do not conflict, that partial downloads are
 156     properly evicted without leaving temporary files, and that access
 157     rights are maintained.
 158
 159     -h  This help text.
 160
 161     -b BACKEND
 162         Wrap the specified BACKEND. Known backends are:
 163             bzr     Bazaar
 164             cp      Local files
 165             cvs     Concurrent Versions System
 166             git     Git
 167             hg      Mercurial
 168             scp     Secure copy
 169             svn     Subversion
 170             wget    HTTP download
 171
 172     -o FILE
 173         Store the downloaded archive in FILE.
 174
 175     -H FILE
 176         Use FILE to read hashes from, and check them against the downloaded
 177         archive.
 178
 179   Exit status:
 180     0   if OK
 181     !0  in case of error
 182
 183 ENVIRONMENT
 184
 185     BUILD_DIR
 186         The path to Buildroot's build dir
 187 _EOF_
 188 }
 189
 190 trace()  { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
 191 warn()   { trace "${@}" >&2; }
 192 errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
 193 error()  { errorN 1 "${@}"; }
 194
 195 my_name="${0##*/}"
 196 main "${@}"