crawler-bin/idokep-offline.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 httrack -%v2 --continue --include-query-string=0 --purge-old=0 --keep-alive --can-go-down --can-go-up --stay-on-same-dir \
   6         --depth=1 -%P --near --index=0 --path . \
   7         -A10000000 -%c100 --update \
   8         https://www.idokep.hu/idojaras/Mak%C3%B3 https://www.idokep.hu/elorejelzes/Mak%C3%B3 \
   9         '-*' '+*.idokep.hu/*' '-www.idokep.hu/keptar/users/*' '-www.idokep.hu/keptar/income/vago/*'
  10
  11 perl -pe 's{^<html.*}{<html>}' -i index.html
  12
  13 cd www.idokep.hu
  14
  15 cat css/*.css | perl -ne 'for $m (m{url\("?(?!"?data:)(.+?)[?#")]}g){ print "$m\n" }' |\
  16 sd '^\.\./+' '' |\
  17 sd '^/+' '' |\
  18 while read path
  19 do
  20         [ ! -s "$path" ] && echo "$path"
  21 done |\
  22 sort -u |\
  23 rcmod any=0 wget --base=https://www.idokep.hu/ -i - --no-host-directories --force-directories
  24
  25
  26 export banner_text_start='<!-- banner_text start -->'
  27 export banner_text_end='<!-- banner_text end -->'
  28
  29 for htmlpath in idojaras/Makó.html elorejelzes/Makó.html
  30 do
  31         mtime_ts=`stat -c %Y "$htmlpath"`
  32         mtime_dt=`date -d@$mtime_ts`
  33         export banner_text="<div class=\"row\"><div class=\"col\" style=\"font-size: 20px; color: black;\">Frissítve ekkor: $mtime_dt</div></div>"
  34         perl -pe 's{(id="menubarDesktop".*?)($ENV{banner_text_start}(.+)$ENV{banner_text_end}|)$}{$1$ENV{banner_text_start}$ENV{banner_text}$ENV{banner_text_end}}' -i "$htmlpath"
  35 done