batchget-puf

   1 #!/usr/bin/perl -w
   2 ###APPNAME:     batchget
   3 ###APPAUTHOR:   xiaoranzzz
   4 ###APPDATE:     Fri Sep 28 12:31:18 2007
   5 ###APPVER:      0.2
   6 ###APPDESC:     Download URLS,using simdo
   7 ###APPUSAGE:    [OPTIONS...] [Task Name]
   8 ###APPEXAMPLE:  cat url.lst | batchget
   9 ###APPOPTION:   -m:Max number of simultanous tasks|-n:Task name|-a:Auto rename|-d:Create directories|-e:extension name for autorename|-w:Working directory|-r:Refer URL|-h:Display this text|-b:cookie|-L:Logging programe|-i:Auto rename by index
  10
  11 use strict;
  12 use lib $ENV{XR_PERL_MODULE_DIR};
  13 use Getopt::Std;
  14
  15 #BEGIN {srand(time());}
  16
  17 my %opts;
  18 getopts('hm:n:ade:w:r:L:b:i',\%opts);
  19 exit(system("formathelp",$0,"-n")) if($opts{"h"});
  20
  21 my $def_mul=3;
  22 my $createdir = $opts{"d"} ? $opts{"d"} : 0;
  23 my $muldown   = $opts{"m"} ? $opts{"m"} : $def_mul;
  24 my $taskname  = $opts{"n"} ? $opts{"n"} : "";
  25 my $autoname  = $opts{"a"} ? $opts{"a"} : 0;
  26 my $extname   = $opts{"e"} ? $opts{"e"} : "";
  27 my $workdir   = $opts{"w"} ? $opts{"w"} : "";
  28 my $refer     = $opts{"r"} ? $opts{"r"} : "";
  29 my $logger    = $opts{"L"} ? $opts{"L"} : "";
  30 my $cookie    = $opts{"b"} ? $opts{"b"} : "";
  31 my $number    = $opts{"i"} ? $opts{"i"} : "";
  32 $autoname="true" if($number);
  33
  34 $taskname = shift unless($taskname and @ARGV);
  35 $taskname = "" unless($taskname);
  36 $muldown = 1 if( $muldown<1);
  37
  38 if($workdir) {
  39     mkdir $workdir unless(-d $workdir);
  40     $workdir .= "/" unless($workdir =~ /\/$/);
  41 #    chdir($workdir);
  42 }
  43
  44 my @URLS;
  45 my $count=0;
  46 while(<STDIN>) {
  47     chomp;
  48     push @URLS,$_;
  49     $count++;
  50 #    print STDERR ("\r$count URLS enqueued...                 ");
  51 }
  52 #print STDERR ("\n");
  53 exit 0 unless($count);
  54
  55 my $index;
  56
  57 sub Uniqname($);
  58 sub Uniqname($) {
  59     my $ext =shift;
  60     my $max = 10000000000;
  61     my $n1 = int (rand($max));
  62     my $n2 = log($max / $n1)/log(10);
  63     my $result=$n1 . "0"x$n2 . $ext;
  64     return $result unless(-f $result);
  65     return Uniqname($ext);
  66 }
  67 sub GetFilename_Auto {
  68     my $URL=shift;
  69     #my $num=shift;
  70     my $result;
  71     #my $ext=$extname;
  72     #($ext=$URL) =~ s/^.*\.([^\.]*)$/.$1/ unless($ext);
  73     #$result=$num ? "$num$ext" : Uniqname($ext);
  74     $result = $URL;
  75     $result =~ s/^.*:\/\///;
  76     $result =~ s/\//_/g;
  77     $result =~ s/&//g;
  78     if($createdir) {
  79         my $dirname=$URL;
  80         $dirname =~ s/^.*:\/*[^\/]*\///;
  81         $dirname =~ s/\/[^\/]*//;
  82         $dirname .= "/" if($dirname);
  83         $result = $dirname . $result;
  84     }
  85     return $result;
  86 }
  87 sub GetFilename_NoAuto {
  88     my $result=shift;
  89     if($createdir) {
  90         $result =~ s/^.*:\/*[^\/]*\///;
  91     }
  92     else {
  93         $result =~ s/^.*\///;
  94     }
  95     return $result;
  96 }
  97
  98 sub pushArgPair(\@$$) {
  99     my $DARG=shift;
 100     my $key=shift;
 101     my $value=shift;
 102     if($value) {
 103         push @{$DARG},$key;
 104         push @{$DARG},$value;
 105     }
 106 }
 107
 108 if($cookie) {
 109     system("mkcookie '$cookie' >download.cookie");
 110     $cookie="download.cookie";
 111 }
 112
 113
 114 my @DARGS=("-c","-u","-nb","-nf","-lc",10,"-ll",5,"-U","Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)","-Tc",20,"-t",2);#,"-Tl",15,"-Tc",20);
 115 push @DARGS,"-P",$workdir if($workdir);
 116 #open FO,"|-","puf",@DARGS,"-i","-" or die("$!\n");
 117 print STDERR "Batchget enqueuing tasks...\n";
 118 foreach my $URL(@URLS) {
 119     $index++;
 120     my $curname= $taskname ? "[$taskname $index/$count]" : "[$index/$count]";
 121     my $stridx = "0" x (length($count)-length($index)+1) . $index if($number);
 122     my $filename= $autoname ? GetFilename_Auto($URL,$stridx) : GetFilename_NoAuto($URL);
 123     print STDERR "\renqueue $curname : $URL";
 124     push @DARGS,$URL;
 125 #  print FO "$URL\n" if($URL);
 126 }
 127 print STDERR "\n";
 128 print STDERR "Batchget starting download(puf used)...\n",$workdir ?  "$workdir>\n" : "";
 129 #close FO;
 130 exec ('puf',@DARGS);
 131 #close FO;
 132
 133