mktar: Use `wc` instead of `du` in summary message
[sunny256-utils.git] / grabber
blob9108cacdb7712f47ab35a14c019bcf1b9f7e5e1f
1 #!/usr/bin/env bash
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License version 2, June 1991.
6 # This program is distributed in the hope that it will be useful,
7 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # GNU General Public License for more details.
11 # You should have received a copy of the GNU General Public License
12 # along with this program. If not, see <https://www.gnu.org/licenses/gpl-2.0.html>,
13 # or in pdf format at <http://www.dhampir.no/stuff/private/gpl-2.0.pdf>
15 # Copyright 2014 - Øyvind 'bolt' Hvidsten <bolt@dhampir.no>
17 # Description:
19 # grabber is a set of utility functions for fetching webpages using curl or wget, logging in, keeping cookies and so on.
21 # It requires curl or wget, xxd, tr, cat and sed. "tempfile" is also recommended, though the script should manage
22 # to generate tempfiles itself.
24 # Generally, this script will be sourced by other bash scripts in order to simplify and unify such activities.
27 # Usage:
29 # Source this script and use its functions.
30 # If your script has a cleanup trap on EXIT, make sure to specifically call grabber_cleanup as well.
33 # Version history:
35 # 2014-10-19: Initial version
36 # 2015-05-15: Improved curl timeout options and renamed agent, timeout and retries options
37 # 2015-05-16: Renamed all functions intended for internal use to avoid interfering with outside scripts
38 # 2015-08-22: Added support for $grab_method
39 # 2015-08-26: Added support for $grab_auth
40 # 2016-01-16: Renamed "encode" function to match shellfunc naming
44 # Check bash version
45 if ((BASH_VERSINFO[0] < 4)); then
46 echo "grabber uses features only found in bash 4 or above. Unable to continue execution." >&2
47 exit 1
51 # Check for required programs
53 ! type "${GRABBER:-curl}" &>/dev/null ||
54 ! type "xxd" &>/dev/null ||
55 ! type "tr" &>/dev/null ||
56 ! type "cat" &>/dev/null ||
57 ! type "sed" &>/dev/null
58 then
59 echo "One or more required programs are missing!" >&2
60 echo "Please ensure ${GRABBER:-curl}, xxd, tr, cat and sed are all installed and in \$PATH" >&2
61 exit 1
65 # How to grab pages
66 function _grabber_grab
68 echo "$@" >"$f_cmd"
69 "$@"
71 function grab
73 >"$f_stdout"
74 >"$f_stderr"
75 local i
76 local agent=${GRABBER_AGENT:-"Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0"}
77 local timeout=${GRABBER_TIMEOUT:-30}
78 local retries=${GRABBER_RETRIES:-3}
80 case "${GRABBER:-curl}" in
81 curl)
82 _grabber_grab curl \
83 --location \
84 --insecure \
85 --cookie $f_cookies \
86 --cookie-jar $f_cookies \
87 --speed-limit 1 \
88 --speed-time $timeout \
89 --connect-timeout $timeout \
90 --retry $retries \
91 --user-agent "$agent" \
92 ${grab_proto:+-"$grab_proto"} \
93 ${grab_data:+--data "$grab_data"} \
94 ${grab_ref:+--referer "$grab_ref"} \
95 ${grab_head:+--header "$grab_head"} \
96 ${grab_method:+--request "$grab_method"} \
97 ${grab_auth:+--user "$grab_auth"} \
98 "$@" >"$f_stdout" 2>"$f_stderr"
100 wget)
101 _grabber_grab wget -O- \
102 --no-check-certificate \
103 --load-cookies=$f_cookies \
104 --save-cookies=$f_cookies \
105 --keep-session-cookies \
106 --timeout $timeout \
107 --tries $retries \
108 --user-agent="$agent" \
109 ${grab_proto:+-"$grab_proto"} \
110 ${grab_data:+--post-data="$grab_data"} \
111 ${grab_ref:+--referer="$grab_ref"} \
112 ${grab_head:+--header="$grab_head"} \
113 ${grab_method:+--method="$grab_method"} \
114 ${grab_auth:+--user="${grab_auth%%:*}" --password="${grab_auth#*:}"} \
115 "$@" >"$f_stdout" 2>"$f_stderr"
118 echo "Fatal script error - grabber configuration fubar" >&2
119 exit 1
121 esac
122 } && grab_status=$? || grab_status=$?
123 (( grab_status )) || return 0
124 grabber_fail
128 # Check for string in output
129 function found
131 grep -q "$@" "$f_stdout"
133 function errfound
135 grep -q "$@" "$f_stderr"
137 function assert
139 if found "$@"; then
140 _grabber_silent || echo "OK"
141 else
142 _grabber_silent || echo "Failed!"
143 echo "Exiting due to failed assertion (${@})" >&2
144 _grabber_debugfail
145 ${GRABBER_EXIT:-true} && exit 1 || return 1
150 # Simple URL encoding - just encode everything
151 function sf_urlencode
153 local encoded=$(xxd -plain <<<"${@:-$(cat)}" | tr -d '\n' | sed 's/\(..\)/%\1/g')
154 echo "${encoded%\%0a}"
158 # Debug printing
159 function grabber_debug
161 ! ${DEBUG:-false} || echo "Debug: $@" >&2
165 # Make a temporary file
166 type "tempfile" &>/dev/null && _grabber_tempfile=false || _grabber_tempfile=true
167 function _grabber_tempfile
169 umask 0077
170 if $_grabber_tempfile; then
171 file="/tmp/${_scriptname}-$RANDOM"
172 set -o noclobber
173 { >"$file"; } &>/dev/null &&
174 echo "$file"
175 else
176 tempfile
181 # Feedback
182 function _grabber_silent
184 ${GRABBER_SILENT:-false}
186 function grabber_task
188 _grabber_silent || echo -n "$@... "
190 function grabber_ok
192 _grabber_silent || echo "OK"
194 function _grabber_debugfail
196 ${DEBUG:-false} || return
197 echo >&2
198 echo "f_stdout:" >&2
199 cat "$f_stdout" >&2
200 echo >&2
201 echo "f_stderr:" >&2
202 cat "$f_stderr" >&2
203 echo >&2
204 echo "f_cookies:"
205 cat "$f_cookies" >&2
206 echo >&2
207 echo "f_cmd:" >&2
208 cat "$f_cmd" >&2
210 function grabber_fail
212 _grabber_silent || echo "Failed!"
213 if [[ -n "$@" ]]; then
214 _grabber_silent || echo "$@" >&2
215 else
216 _grabber_silent || echo "General failure" >&2
218 _grabber_silent || _grabber_debugfail
219 ${GRABBER_EXIT:-true} && exit 1 || return 1
223 # Temp file generation and cleanup
224 function grabber_cleanup
226 grabber_task "Cleanup"
228 [[ -n "${f_cookies:-}" ]] && [[ -e "$f_cookies" ]] && rm "$f_cookies" &&
229 [[ -n "${f_stdout:-}" ]] && [[ -e "$f_stdout" ]] && rm "$f_stdout" &&
230 [[ -n "${f_stderr:-}" ]] && [[ -e "$f_stderr" ]] && rm "$f_stderr" &&
231 [[ -n "${f_cmd:-}" ]] && [[ -e "$f_cmd" ]] && rm "$f_cmd"
232 then
233 grabber_ok
234 else
235 grabber_fail
238 trap grabber_cleanup EXIT
241 grabber_task "Generating grabber cookie file"
242 if f_cookies=$(_grabber_tempfile) && [[ -n "$f_cookies" ]] && [[ -e "$f_cookies" ]]; then
243 _grabber_silent || echo "$f_cookies"
244 else
245 grabber_fail
247 grabber_task "Generating grabber stdout file"
248 if f_stdout=$(_grabber_tempfile) && [[ -n "$f_stdout" ]] && [[ -e "$f_stdout" ]]; then
249 _grabber_silent || echo "$f_stdout"
250 else
251 grabber_fail
253 grabber_task "Generating grabber stderr file"
254 if f_stderr=$(_grabber_tempfile) && [[ -n "$f_stderr" ]] && [[ -e "$f_stderr" ]]; then
255 _grabber_silent || echo "$f_stderr"
256 else
257 grabber_fail
259 grabber_task "Generating grabber cmd file"
260 if f_cmd=$(_grabber_tempfile) && [[ -n "$f_cmd" ]] && [[ -e "$f_cmd" ]]; then
261 _grabber_silent || echo "$f_cmd"
262 else
263 grabber_fail
267 # vim: tabstop=4:softtabstop=4:shiftwidth=4:noexpandtab