download

   1 #!/usr/bin/perl -w
   2 # $Id$
   3 ###APPNAME:     download
   4 ###APPAUTHOR:   geek
   5 ###APPDATE:     Fri Sep 28 08:31:24 2007
   6 ###APPVER:      0.2
   7 ###APPDESC:     A downloader,nothing else.
   8 ###APPUSAGE:    [options] URL
   9 ###APPEXAMPLE:  download www.google.com/index.html
  10 ###APPOPTION:   -v:Verbose Output|-u:URL to download|-s:Filename for output|-d:Create directories|-n:Task name|-r:Referer URL|-b:cookie file|-l:Enable Logging|-a:Auto rename if file exists|-p:Downloader - wget or curl-m:Max time for downloading|-f:Force over writting
  11 use strict;
  12 use lib $ENV{XR_PERL_MODULE_DIR};
  13
  14 use MyPlace::Script::Usage qw/help_required help_even_empty/;
  15 use Cwd;
  16 use Getopt::Std;
  17 use MyPlace::Script::Message;
  18
  19 exit 0 if(help_even_empty($0,@ARGV));
  20
  21 my $proxy = '127.0.0.1:9050';
  22 my $blocked_host = 'wretch\.cc|facebook\.com|fbcdn\.net';
  23 my $BLOCKED_EXP = qr/^[^\/]+:\/\/[^\/]*(?:$blocked_host)(?:\/?|\/.*)$/;
  24 my @WGET = qw{
  25     wget --user-agent Mozilla/5.0 --connect-timeout 15 -q --progress bar
  26 };
  27 my @CURL = qw{
  28         curl --fail --globoff --location --user-agent Mozilla/5.0 --progress-bar --create-dirs
  29 };
  30 sub log($$) {
  31     my $text=shift;
  32     my $fn=shift;
  33     open FO,">>",$fn or return;
  34     print FO $text;
  35     close FO;
  36 }
  37
  38 sub build_cmdline {
  39     my($name,$url,$saveas,$refer,$cookie,$verbose,$maxtime) = @_;
  40     return undef unless($url);
  41     my @result;
  42     if($name =~ /^wget$/i) {
  43         push @result,@WGET;
  44         push @result,"--referer",$refer ? $refer : $url;
  45         push @result,"--output-document",$saveas if($saveas);
  46         push @result,"--load-cookie",$cookie if(-f $cookie);
  47         push @result,"--save-cookie",$cookie if($cookie);
  48         push @result,'--read-timeout',$maxtime if($maxtime);
  49         push @result,$url;
  50     }
  51     else {
  52         push @result,@CURL;
  53         push @result,"--url",$url;
  54         push @result,"--referer",$refer ? $refer : $url;
  55         push @result,"--output",$saveas if($saveas);
  56         push @result,"--cookie",$cookie if(-f $cookie);
  57         push @result,"--cookie-jar",$cookie if($cookie);
  58         push @result,"--max-time",$maxtime if($maxtime);
  59         if($url =~ $BLOCKED_EXP) {
  60             app_message "USE PROXY $proxy\n";
  61             push @result,"--socks5-hostname",$proxy;
  62         }
  63     }
  64     return @result;
  65 }
  66
  67 sub process {
  68     my $taskname=shift;
  69     my $cmdline=shift;
  70     my $retry = shift || 2;
  71     my $r=0;
  72     while($retry) {
  73         $retry--;
  74         $r=system(@{$cmdline});
  75         return 0 if($r==0);
  76         return 2 if($r==2); #/KILL,TERM,USERINT;
  77         $r = $r>>8;
  78         #2 =>
  79         #22 => Request Error 404,403
  80         #56 => Recv failure: Connection reset by peer
  81         return $r if($r == 2 or $r == 22 or $r == 56 or $r == 6);
  82         app_warning "\rdownload:error($r), wait 1 second,retry $taskname\n";
  83         sleep 1;
  84     }
  85     return 1;
  86 }
  87
  88
  89 my $OptFlag='m:lvu:s:dn:r:b:ap:f';
  90 my %OPT;
  91 getopts($OptFlag,\%OPT);
  92
  93 my $verbose= $OPT{"v"} ? $OPT{"v"} : 0;
  94 my $url= $OPT{"u"} ? $OPT{"u"} : $ARGV[@ARGV-1];
  95 my $saveas= $OPT{"s"} ? $OPT{"s"} : "";
  96 my $createdir= $OPT{"d"} ? $OPT{"d"} : 0;
  97 my $name= $OPT{"n"} ? $OPT{"n"} : "";
  98 my $refer= $OPT{"r"} ? $OPT{"r"} : "";
  99 my $cookie= $OPT{b} ? $OPT{b} : "";
 100 my $logging= $OPT{l} ? $OPT{l} : "";
 101 my $autorename = $OPT{a} ? $OPT{a} : "";
 102 my $maxtime = $OPT{m} ? $OPT{m} : undef;
 103 my $force = $OPT{f} ? $OPT{f} : undef;
 104 #my $downloader = $OPT{p} ? $OPT{p} : "wget";
 105 my $downloader = $OPT{p} ? $OPT{p} : "curl";
 106 my $FAILLOG="download.failed";
 107 my $DOWNLOADLOG="download.log";
 108
 109 if ($url !~ m/^\w+:\/\// ) {
 110     app_message("invaild URL:\"",$url,"\"\n");
 111     exit 1;
 112 }
 113
 114 $url =~ s/\ /%20/g;
 115 $refer=$url unless($refer);
 116 if($createdir && !$saveas) {
 117     my $filename=$url;
 118     $filename =~ s/^\w+:\/+[^\/]*\/+//;
 119     $filename =~ s/^[^\?]*\?[^\/]*\/+//g;
 120     $saveas=$filename;
 121 }
 122 if(!$saveas) {
 123     my $basename=$url;
 124     $basename =~ s/^.*\///g;
 125     $basename = "index.htm" unless($basename);
 126     $saveas=$basename;
 127 }
 128 if($saveas =~ m/\/$/) {
 129     $saveas .= "index.htm";
 130 }
 131 if($saveas and $autorename and -f $saveas) {
 132     my $base = $saveas;
 133     my $ext;
 134     if($saveas =~ m/^(.*)(\.[^\.]*)$/) {
 135         $base = $1;
 136         $ext = $2;
 137     }
 138     use MyPlace::Filename qw/get_uniqname/;
 139     $saveas = get_uniqname($base,$ext);
 140 }
 141
 142 if($verbose) {
 143     app_message(sprintf("%s\n%-8s: %s\n%-8s: %s\n%-8s: %s\n",
 144             $name ? "\n$name" : "",
 145             "URL",$url,
 146             "SaveAs",$saveas,
 147             "Refer",$refer));
 148 }
 149 else {
 150     app_message "$name$url\t[starting]\n";
 151 }
 152
 153 if ((!$force) and -f "$saveas" ) {
 154     app_warning "$saveas exists\t[canceled]\n";
 155     exit 0;
 156 }
 157
 158 if($cookie) {
 159     if(!-f $cookie) {
 160         app_message "creating cookie for $url...\n";
 161         my @match = $url =~ /^(http:\/\/[^\/]+)\//;
 162         if(@match) {
 163             my $domain=$match[0];
 164             system("curl --url '$domain' -c '$cookie' -o '/dev/null'");
 165         }
 166     }
 167 }
 168
 169 my $saveas_temp = "$saveas.downloading";
 170 my @cmdline = build_cmdline($downloader,$url,$saveas_temp,$refer,$cookie,$verbose,$maxtime);
 171 my $r=process("$name$url",\@cmdline,2);
 172
 173 if($r==0 and -f $saveas_temp) {
 174     unlink ($saveas) if(-f $saveas);
 175     rename($saveas_temp,$saveas) or die("$!\n");
 176     &log("$url->$saveas\n","$DOWNLOADLOG") if($logging);;
 177     app_ok "$name$saveas\t[completed]\n";
 178     exit 0;
 179 }
 180 elsif($r==2) {
 181     unlink $saveas_temp if(-f $saveas_temp);
 182     app_warning "$name$url\t[killed]\n";
 183     exit 2;
 184 }
 185 else {
 186     unlink $saveas_temp if(-f $saveas_temp);
 187     app_error "$name$url\t[failed]\n";
 188     &log("$url->$saveas\n","$FAILLOG") if($logging);;
 189     exit 0;
 190 }
 191