grabber

   1 #!/usr/bin/env bash
   2
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License version 2, June 1991.
   5 #
   6 # This program is distributed in the hope that it will be useful,
   7 # but WITHOUT ANY WARRANTY; without even the implied warranty of
   8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9 # GNU General Public License for more details.
  10 #
  11 # You should have received a copy of the GNU General Public License
  12 # along with this program. If not, see <https://www.gnu.org/licenses/gpl-2.0.html>,
  13 # or in pdf format at <http://www.dhampir.no/stuff/private/gpl-2.0.pdf>
  14
  15 # Copyright 2014 - Øyvind 'bolt' Hvidsten   <bolt@dhampir.no>
  16
  17 # Description:
  18 #
  19 # grabber is a set of utility functions for fetching webpages using curl or wget, logging in, keeping cookies and so on.
  20 #
  21 # It requires curl or wget, xxd, tr, cat and sed. "tempfile" is also recommended, though the script should manage
  22 # to generate tempfiles itself.
  23 #
  24 # Generally, this script will be sourced by other bash scripts in order to simplify and unify such activities.
  25 #
  26
  27 # Usage:
  28 #
  29 # Source this script and use its functions.
  30 # If your script has a cleanup trap on EXIT, make sure to specifically call grabber_cleanup as well.
  31 #
  32
  33 # Version history:
  34 #
  35 # 2014-10-19: Initial version
  36 # 2015-05-15: Improved curl timeout options and renamed agent, timeout and retries options
  37 # 2015-05-16: Renamed all functions intended for internal use to avoid interfering with outside scripts
  38 # 2015-08-22: Added support for $grab_method
  39 # 2015-08-26: Added support for $grab_auth
  40 # 2016-01-16: Renamed "encode" function to match shellfunc naming
  41 #
  42
  43
  44 # Check bash version
  45 if ((BASH_VERSINFO[0] < 4)); then
  46         echo "grabber uses features only found in bash 4 or above. Unable to continue execution." >&2
  47         exit 1
  48 fi
  49
  50
  51 # Check for required programs
  52 if
  53         ! type "${GRABBER:-curl}" &>/dev/null ||
  54         ! type "xxd" &>/dev/null ||
  55         ! type "tr" &>/dev/null ||
  56         ! type "cat" &>/dev/null ||
  57         ! type "sed" &>/dev/null
  58 then
  59         echo "One or more required programs are missing!" >&2
  60         echo "Please ensure ${GRABBER:-curl}, xxd, tr, cat and sed are all installed and in \$PATH" >&2
  61         exit 1
  62 fi
  63
  64
  65 # How to grab pages
  66 function _grabber_grab
  67 {
  68         echo "$@" >"$f_cmd"
  69         "$@"
  70 }
  71 function grab
  72 {
  73         >"$f_stdout"
  74         >"$f_stderr"
  75         local i
  76         local agent=${GRABBER_AGENT:-"Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0"}
  77         local timeout=${GRABBER_TIMEOUT:-30}
  78         local retries=${GRABBER_RETRIES:-3}
  79         {
  80                 case "${GRABBER:-curl}" in
  81                         curl)
  82                                 _grabber_grab curl \
  83                                         --location \
  84                                         --insecure \
  85                                         --cookie $f_cookies \
  86                                         --cookie-jar $f_cookies \
  87                                         --speed-limit 1 \
  88                                         --speed-time $timeout \
  89                                         --connect-timeout $timeout \
  90                                         --retry $retries \
  91                                         --user-agent "$agent" \
  92                                         ${grab_proto:+-"$grab_proto"} \
  93                                         ${grab_data:+--data "$grab_data"} \
  94                                         ${grab_ref:+--referer "$grab_ref"} \
  95                                         ${grab_head:+--header "$grab_head"} \
  96                                         ${grab_method:+--request "$grab_method"} \
  97                                         ${grab_auth:+--user "$grab_auth"} \
  98                                         "$@" >"$f_stdout" 2>"$f_stderr"
  99                         ;;
 100                         wget)
 101                                 _grabber_grab wget -O- \
 102                                         --no-check-certificate \
 103                                         --load-cookies=$f_cookies \
 104                                         --save-cookies=$f_cookies \
 105                                         --keep-session-cookies \
 106                                         --timeout $timeout \
 107                                         --tries $retries \
 108                                         --user-agent="$agent" \
 109                                         ${grab_proto:+-"$grab_proto"} \
 110                                         ${grab_data:+--post-data="$grab_data"} \
 111                                         ${grab_ref:+--referer="$grab_ref"} \
 112                                         ${grab_head:+--header="$grab_head"} \
 113                                         ${grab_method:+--method="$grab_method"} \
 114                                         ${grab_auth:+--user="${grab_auth%%:*}" --password="${grab_auth#*:}"} \
 115                                         "$@" >"$f_stdout" 2>"$f_stderr"
 116                         ;;
 117                         *)
 118                                 echo "Fatal script error - grabber configuration fubar" >&2
 119                                 exit 1
 120                         ;;
 121                 esac
 122         } && grab_status=$? || grab_status=$?
 123         (( grab_status )) || return 0
 124         grabber_fail
 125 }
 126
 127
 128 # Check for string in output
 129 function found
 130 {
 131         grep -q "$@" "$f_stdout"
 132 }
 133 function errfound
 134 {
 135         grep -q "$@" "$f_stderr"
 136 }
 137 function assert
 138 {
 139         if found "$@"; then
 140                 _grabber_silent || echo "OK"
 141         else
 142                 _grabber_silent || echo "Failed!"
 143                 echo "Exiting due to failed assertion (${@})" >&2
 144                 _grabber_debugfail
 145                 ${GRABBER_EXIT:-true} && exit 1 || return 1
 146         fi
 147 }
 148
 149
 150 # Simple URL encoding - just encode everything
 151 function sf_urlencode
 152 {
 153         local encoded=$(xxd -plain <<<"${@:-$(cat)}" | tr -d '\n' | sed 's/\(..\)/%\1/g')
 154         echo "${encoded%\%0a}"
 155 }
 156
 157
 158 # Debug printing
 159 function grabber_debug
 160 {
 161         ! ${DEBUG:-false} || echo "Debug: $@" >&2
 162 }
 163
 164
 165 # Make a temporary file
 166 type "tempfile" &>/dev/null && _grabber_tempfile=false || _grabber_tempfile=true
 167 function _grabber_tempfile
 168 {(
 169         umask 0077
 170         if $_grabber_tempfile; then
 171                 file="/tmp/${_scriptname}-$RANDOM"
 172                 set -o noclobber
 173                 { >"$file"; } &>/dev/null &&
 174                 echo "$file"
 175         else
 176                 tempfile
 177         fi
 178 )}
 179
 180
 181 # Feedback
 182 function _grabber_silent
 183 {
 184         ${GRABBER_SILENT:-false}
 185 }
 186 function grabber_task
 187 {
 188         _grabber_silent || echo -n "$@... "
 189 }
 190 function grabber_ok
 191 {
 192         _grabber_silent || echo "OK"
 193 }
 194 function _grabber_debugfail
 195 {
 196         ${DEBUG:-false} || return
 197         echo >&2
 198         echo "f_stdout:" >&2
 199         cat "$f_stdout" >&2
 200         echo >&2
 201         echo "f_stderr:" >&2
 202         cat "$f_stderr" >&2
 203         echo >&2
 204         echo "f_cookies:"
 205         cat "$f_cookies" >&2
 206         echo >&2
 207         echo "f_cmd:" >&2
 208         cat "$f_cmd" >&2
 209 }
 210 function grabber_fail
 211 {
 212         _grabber_silent || echo "Failed!"
 213         if [[ -n "$@" ]]; then
 214                 _grabber_silent || echo "$@" >&2
 215         else
 216                 _grabber_silent || echo "General failure" >&2
 217         fi
 218         _grabber_silent || _grabber_debugfail
 219         ${GRABBER_EXIT:-true} && exit 1 || return 1
 220 }
 221
 222
 223 # Temp file generation and cleanup
 224 function grabber_cleanup
 225 {
 226         grabber_task "Cleanup"
 227         if
 228                 [[ -n "${f_cookies:-}" ]] && [[ -e "$f_cookies" ]] && rm "$f_cookies" &&
 229                 [[ -n "${f_stdout:-}" ]] && [[ -e "$f_stdout" ]] && rm "$f_stdout" &&
 230                 [[ -n "${f_stderr:-}" ]] && [[ -e "$f_stderr" ]] && rm "$f_stderr" &&
 231                 [[ -n "${f_cmd:-}" ]] && [[ -e "$f_cmd" ]] && rm "$f_cmd"
 232         then
 233                 grabber_ok
 234         else
 235                 grabber_fail
 236         fi
 237 }
 238 trap grabber_cleanup EXIT
 239
 240
 241 grabber_task "Generating grabber cookie file"
 242 if f_cookies=$(_grabber_tempfile) && [[ -n "$f_cookies" ]] && [[ -e "$f_cookies" ]]; then
 243         _grabber_silent || echo "$f_cookies"
 244 else
 245         grabber_fail
 246 fi
 247 grabber_task "Generating grabber stdout file"
 248 if f_stdout=$(_grabber_tempfile) && [[ -n "$f_stdout" ]] && [[ -e "$f_stdout" ]]; then
 249         _grabber_silent || echo "$f_stdout"
 250 else
 251         grabber_fail
 252 fi
 253 grabber_task "Generating grabber stderr file"
 254 if f_stderr=$(_grabber_tempfile) && [[ -n "$f_stderr" ]] && [[ -e "$f_stderr" ]]; then
 255         _grabber_silent || echo "$f_stderr"
 256 else
 257         grabber_fail
 258 fi
 259 grabber_task "Generating grabber cmd file"
 260 if f_cmd=$(_grabber_tempfile) && [[ -n "$f_cmd" ]] && [[ -e "$f_cmd" ]]; then
 261         _grabber_silent || echo "$f_cmd"
 262 else
 263         grabber_fail
 264 fi
 265
 266
 267 # vim: tabstop=4:softtabstop=4:shiftwidth=4:noexpandtab