crawler-bin/waybackmachine-dl

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 url=$1
   6
   7
   8 curl -sS "http://web.archive.org/web/timemap/link/$url" |\
   9 while read -r link parameters
  10 do
  11         # strip leading "<" and trailing ">;"
  12         memento_url=${link:1: -2}
  13
  14         if expr "$parameters" : '.*memento' >/dev/null
  15         then
  16                 datetime=`expr "$parameters" : '.*datetime="\([^"]\+\)'`
  17                 echo "$memento_url $datetime"
  18         fi
  19 done |\
  20 sort -r |\
  21 while read -r memento_url datetime
  22 do
  23         echo "$datetime"
  24         snapshot=`date +%F-%T -d "$datetime"`
  25         mkdir -p "$snapshot"
  26         (
  27                 set -e
  28                 cd "$snapshot"
  29                 # "||true" -> ignore http errors like upstream 404
  30                 wget -c "$memento_url" || true
  31         )
  32 done