support/download/dl-wrapper

   1 #!/usr/bin/env bash
   2
   3 # This script is a wrapper to the other download backends.
   4 # Its role is to ensure atomicity when saving downloaded files
   5 # back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
   6 # failed downloads.
   7 #
   8 # Call it with -h to see some help.
   9
  10 # To avoid cluttering BR2_DL_DIR, we download to a trashable
  11 # location, namely in $(BUILD_DIR).
  12 # Then, we move the downloaded file to a temporary file in the
  13 # same directory as the final output file.
  14 # This allows us to finally atomically rename it to its final
  15 # name.
  16 # If anything goes wrong, we just remove all the temporaries
  17 # created so far.
  18
  19 # We want to catch any unexpected failure, and exit immediately.
  20 set -e
  21
  22 main() {
  23     local OPT OPTARG
  24     local backend output hfile recurse quiet
  25
  26     # Parse our options; anything after '--' is for the backend
  27     while getopts :hb:o:H:rq OPT; do
  28         case "${OPT}" in
  29         h)  help; exit 0;;
  30         b)  backend="${OPTARG}";;
  31         o)  output="${OPTARG}";;
  32         H)  hfile="${OPTARG}";;
  33         r)  recurse="-r";;
  34         q)  quiet="-q";;
  35         :)  error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
  36         \?) error "unknown option '%s'\n" "${OPTARG}";;
  37         esac
  38     done
  39     # Forget our options, and keep only those for the backend
  40     shift $((OPTIND-1))
  41
  42     if [ -z "${backend}" ]; then
  43         error "no backend specified, use -b\n"
  44     fi
  45     if [ -z "${output}" ]; then
  46         error "no output specified, use -o\n"
  47     fi
  48
  49     # If the output file already exists and:
  50     # - there's no .hash file: do not download it again and exit promptly
  51     # - matches all its hashes: do not download it again and exit promptly
  52     # - fails at least one of its hashes: force a re-download
  53     # - there's no hash (but a .hash file): consider it a hard error
  54     if [ -e "${output}" ]; then
  55         if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
  56             exit 0
  57         elif [ ${?} -ne 2 ]; then
  58             # Do not remove the file, otherwise it might get re-downloaded
  59             # from a later location (i.e. primary -> upstream -> mirror).
  60             # Do not print a message, check-hash already did.
  61             exit 1
  62         fi
  63         rm -f "${output}"
  64         warn "Re-downloading '%s'...\n" "${output##*/}"
  65     fi
  66
  67     # tmpd is a temporary directory in which backends may store intermediate
  68     # by-products of the download.
  69     # tmpf is the file in which the backends should put the downloaded content.
  70     # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
  71     # $(BR2_DL_DIR)
  72     # We let the backends create tmpf, so they are able to set whatever
  73     # permission bits they want (although we're only really interested in
  74     # the executable bit.)
  75     tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
  76     tmpf="${tmpd}/output"
  77
  78     # Helpers expect to run in a directory that is *really* trashable, so
  79     # they are free to create whatever files and/or sub-dirs they might need.
  80     # Doing the 'cd' here rather than in all backends is easier.
  81     cd "${tmpd}"
  82
  83     # If the backend fails, we can just remove the temporary directory to
  84     # remove all the cruft it may have left behind. Then we just exit in
  85     # error too.
  86     if ! "${OLDPWD}/support/download/${backend}" ${quiet} ${recurse} "${tmpf}" "${@}"; then
  87         rm -rf "${tmpd}"
  88         exit 1
  89     fi
  90
  91     # cd back to free the temp-dir, so we can remove it later
  92     cd "${OLDPWD}"
  93
  94     # Check if the downloaded file is sane, and matches the stored hashes
  95     # for that file
  96     if ! support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
  97         rm -rf "${tmpd}"
  98         exit 1
  99     fi
 100
 101     # tmp_output is in the same directory as the final output, so we can
 102     # later move it atomically.
 103     tmp_output="$(mktemp "${output}.XXXXXX")"
 104
 105     # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
 106     # to users other than the one doing the download (and root, of course).
 107     # This can be problematic when a shared BR2_DL_DIR is used by different
 108     # users (e.g. on a build server), where all users may write to the shared
 109     # location, since other users would not be allowed to read the files
 110     # another user downloaded.
 111     # So, we restore the 'go' access rights to a more sensible value, while
 112     # still abiding by the current user's umask. We must do that before the
 113     # final 'mv', so just do it now.
 114     # Some backends (cp and scp) may create executable files, so we need to
 115     # carry the executable bit if needed.
 116     [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
 117     new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
 118     chmod ${new_mode} "${tmp_output}"
 119
 120     # We must *not* unlink tmp_output, otherwise there is a small window
 121     # during which another download process may create the same tmp_output
 122     # name (very, very unlikely; but not impossible.)
 123     # Using 'cp' is not reliable, since 'cp' may unlink the destination file
 124     # if it is unable to open it with O_WRONLY|O_TRUNC; see:
 125     #   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
 126     # Since the destination filesystem can be anything, it might not support
 127     # O_TRUNC, so 'cp' would unlink it first.
 128     # Use 'cat' and append-redirection '>>' to save to the final location,
 129     # since that is the only way we can be 100% sure of the behaviour.
 130     if ! cat "${tmpf}" >>"${tmp_output}"; then
 131         rm -rf "${tmpd}" "${tmp_output}"
 132         exit 1
 133     fi
 134     rm -rf "${tmpd}"
 135
 136     # tmp_output and output are on the same filesystem, so POSIX guarantees
 137     # that 'mv' is atomic, because it then uses rename() that POSIX mandates
 138     # to be atomic, see:
 139     #   http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
 140     if ! mv -f "${tmp_output}" "${output}"; then
 141         rm -f "${tmp_output}"
 142         exit 1
 143     fi
 144 }
 145
 146 help() {
 147     cat <<_EOF_
 148 NAME
 149     ${my_name} - download wrapper for Buildroot
 150
 151 SYNOPSIS
 152     ${my_name} [OPTION]... -- [BACKEND OPTION]...
 153
 154 DESCRIPTION
 155     Wrapper script around different download mechanisms. Ensures that
 156     concurrent downloads do not conflict, that partial downloads are
 157     properly evicted without leaving temporary files, and that access
 158     rights are maintained.
 159
 160     -h  This help text.
 161
 162     -b BACKEND
 163         Wrap the specified BACKEND. Known backends are:
 164             bzr     Bazaar
 165             cp      Local files
 166             cvs     Concurrent Versions System
 167             git     Git
 168             hg      Mercurial
 169             scp     Secure copy
 170             svn     Subversion
 171             wget    HTTP download
 172
 173     -o FILE
 174         Store the downloaded archive in FILE.
 175
 176     -H FILE
 177         Use FILE to read hashes from, and check them against the downloaded
 178         archive.
 179
 180   Exit status:
 181     0   if OK
 182     !0  in case of error
 183
 184 ENVIRONMENT
 185
 186     BUILD_DIR
 187         The path to Buildroot's build dir
 188 _EOF_
 189 }
 190
 191 trace()  { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
 192 warn()   { trace "${@}" >&2; }
 193 errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
 194 error()  { errorN 1 "${@}"; }
 195
 196 my_name="${0##*/}"
 197 main "${@}"