test/make-html.pl

   1 #!/usr/bin/perl
   2 #
   3 # Copyright © 2005 Mozilla Corporation
   4 #
   5 # Permission to use, copy, modify, distribute, and sell this software
   6 # and its documentation for any purpose is hereby granted without
   7 # fee, provided that the above copyright notice appear in all copies
   8 # and that both that copyright notice and this permission notice
   9 # appear in supporting documentation, and that the name of
  10 # Mozilla Corporation not be used in advertising or publicity pertaining to
  11 # distribution of the software without specific, written prior
  12 # permission. Mozilla Corporation makes no representations about the
  13 # suitability of this software for any purpose.  It is provided "as
  14 # is" without express or implied warranty.
  15 #
  16 # MOZILLA CORPORTAION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  17 # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  18 # FITNESS, IN NO EVENT SHALL MOZILLA CORPORATION BE LIABLE FOR ANY SPECIAL,
  19 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  20 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  21 # OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  22 # IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  23 #
  24 # Author: Vladimir Vukicevic <vladimir@pobox.com>
  25 #
  26
  27 use MIME::Base64;
  28
  29 ##
  30 ## Takes all the *.log files in the current directory (or those provided
  31 ## on the command line) and spits out html to stdout that can be used to
  32 ## view all the test results at once.
  33 ##
  34
  35 # show reference images
  36 $config_show_ref = $ENV{'CAIRO_TEST_SHOW_REF'} || 0;
  37
  38 # show fail images
  39 $config_show_fail = $ENV{'CAIRO_TEST_SHOW_FAIL'} || 1;
  40
  41 # show all results, even passes
  42 $config_show_all = $ENV{'CAIRO_TEST_SHOW_ALL'} || 0;
  43
  44 # include test results inline
  45 $config_show_inline = $ENV{'CAIRO_TEST_SHOW_INLINE'} || 0;
  46
  47 # end of config options
  48
  49 $tests = {};
  50 $teststats = {};
  51 $logs = {};
  52
  53 if ($#ARGV >= 0) { @files = @ARGV; } else { @files = <*.log>; }
  54
  55 foreach (<@files>) {
  56   my $testname;
  57   my $out_path, $diff_path, $ref_path;
  58   my $fn = $_;
  59   (open LOG, $fn) || next;
  60   while (<LOG>) {
  61     if (/^OUTPUT: (.*)$/) {
  62       $out_path = $1;
  63       next;
  64     }
  65     if (/^DIFFERENCE: (.*)$/) {
  66       $diff_path = $1;
  67       next;
  68     }
  69     if (/^REFERENCE: (.*)$/) {
  70       $ref_path = $1;
  71       next;
  72     }
  73     next unless /^TEST: (.*) TARGET: (.*) FORMAT: (.*) OFFSET: (.*) SIMILAR: (.*) RESULT: ([A-Z]*).*$/;
  74     $testname = $1 if !defined($testname);
  75     $tests->{$1} = {} unless $tests->{$1};
  76     $tests->{$1}->{$2} = {} unless $tests->{$1}->{$2};
  77     $tests->{$1}->{$2}->{$3} = {} unless $tests->{$1}->{$2}->{$3};
  78     $tests->{$1}->{$2}->{$3}->{$4} = {} unless $tests->{$1}->{$2}->{$3}->{$4};
  79     $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'out'} = $out_path;
  80     $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'diff'} = $diff_path;
  81     $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'ref'} = $ref_path;
  82     $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'result'} = $6;
  83
  84     $teststats->{$2} = {"PASS" => 0, "FAIL" => 0, "XFAIL" => 0, "UNTESTED" => 0, "CRASHED" =>0}
  85       unless $teststats->{$2};
  86     ($teststats->{$2}->{$6})++;
  87
  88     undef $out_path;
  89     undef $diff_path;
  90     undef $ref_path;
  91   }
  92   close LOG;
  93
  94   (open LOG, $fn) || die "I could open it earlier, but I can't now: $!";
  95   {
  96     local $/;
  97     $logs->{$testname} = <LOG>;
  98   }
  99   close LOG;
 100 }
 101
 102 my $targeth = {};
 103 my $formath = {};
 104 my $offseth = {};
 105 my $similarh = {};
 106
 107 foreach my $testname (sort(keys %$tests)) {
 108   my $v0 = $tests->{$testname};
 109   foreach my $targetname (sort(keys %$v0)) {
 110     my $v1 = $v0->{$targetname};
 111
 112     $targeth->{$targetname} = 1;
 113     foreach my $formatname (sort(keys %$v1)) {
 114       my $v2 = $v1->{$formatname};
 115
 116       $formath->{$formatname} = 1;
 117       foreach my $offsetval (sort(keys %$v2)) {
 118         my $v3 = $v2->{$offsetval};
 119
 120         $offseth->{$offsetval} = 1;
 121         foreach my $similarval (sort(keys %$v3)) {
 122           $similarh->{$similarval} = 1;
 123         }
 124       }
 125     }
 126   }
 127 }
 128
 129 my @targets = sort(keys %$targeth);
 130 my @formats = sort(keys %$formath);
 131 my @offsets = sort(keys %$offseth);
 132 my @similars = sort(keys %$similarh);
 133
 134 sub printl {
 135   print @_, "\n";
 136 }
 137
 138 # convert file into a data URI
 139 sub file_to_data {
 140   my ($ctype,$fname) = @_;
 141   my $fdata;
 142   open FILE, $fname || return "data:" . $ctype . ",";
 143   {
 144     local $/;
 145     $fdata = encode_base64(<FILE>);
 146   }
 147   close FILE;
 148   return "data:" . $ctype . ";base64," . $fdata;
 149 }
 150
 151 # convert string into a data URI
 152 sub string_to_data {
 153   my ($ctype,$str) = @_;
 154   $str =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
 155   return "data:" . $ctype . "," . $str;
 156 }
 157
 158 printl '<html><head>';
 159 printl '<title>Cairo Test Results</title>';
 160 printl '<style type="text/css">';
 161 printl 'a img { border: solid 1px #FFF; }';
 162 printl '.PASS { background-color: #0B0; min-width: 1em; }';
 163 printl '.FAIL { background-color: #B00; }';
 164 printl '.XFAIL { background-color: #BB0; }';
 165 printl '.UNTESTED { background-color: #555; }';
 166 printl '.CRASHED { background-color: #F00; color: #FF0; }';
 167 printl '.PASSstr { color: #0B0; }';
 168 printl '.FAILstr { color: #D00; }';
 169 printl '.XFAILstr { color: #BB0; }';
 170 printl '.CRASHEDstr { color: #F00; }';
 171 printl '.UNTESTEDstr { color: #555; }';
 172 printl 'img { max-width: 15em; min-width: 3em; min-height: 3em; margin: 3px; }';
 173 printl 'td { vertical-align: top; }';
 174 printl '</style>';
 175 printl '</script>';
 176 printl '</head>';
 177 printl '<body>';
 178
 179 printl '<table border="1">';
 180 print '<tr><th>Test</th>';
 181
 182 foreach my $target (@targets) {
 183   print '<th>', $target, '</th>';
 184 }
 185 printl '</tr>';
 186
 187 print '<tr><td></td>';
 188
 189 foreach my $target (@targets) {
 190   print '<td>';
 191   print '<span class="PASSstr">', $teststats->{$target}->{"PASS"}, '</span>/';
 192   print '<span class="FAILstr">', $teststats->{$target}->{"FAIL"} + $teststats->{$target}->{"CRASHED"}, '</span>/';
 193   print '<span class="XFAILstr">', $teststats->{$target}->{"XFAIL"}, '</span>/';
 194   print '<span class="UNTESTEDstr">', $teststats->{$target}->{"UNTESTED"}, '</span>';
 195   print '</td>';
 196 }
 197 printl '</tr>';
 198
 199 sub img_for {
 200   my ($fn, $withlink) = @_;
 201
 202   if ($config_show_inline) {
 203     $fn = file_to_data("image/png", $fn);
 204     # never return links, people can just right-click view image,
 205     # and we don't clutter the document
 206     return '<img src="' . $fn . '">';
 207   } else {
 208     if ($withlink) {
 209       return '<a href="' . $fn . '"><img src="' . $fn . '"></a>';
 210     } else {
 211       return '<img src="' . $fn . '">';
 212     }
 213   }
 214 }
 215
 216 foreach my $test (sort(keys %$tests)) {
 217   foreach my $offset (@offsets) {
 218     foreach my $similar (@similars) {
 219       foreach my $format (@formats) {
 220         my $testline = "";
 221
 222         foreach my $target (@targets) {
 223           my $tgtdata = $tests->{$test}->{$target};
 224           if ($tgtdata) {
 225             my $testres = $tgtdata->{$format}->{$offset}->{$similar};
 226             if ($testres) {
 227               my %testfiles;
 228               $testfiles{'out'} = $testres->{'out'};
 229               $testfiles{'diff'} = $testres->{'diff'};
 230               $testfiles{'ref'} = $testres->{'ref'};
 231
 232               $testline .= "<td class=\"$testres->{'result'}\">";
 233               $teststats{$target}{$testres}++;
 234               if ($testres->{'result'} eq "PASS") {
 235                 if ($config_show_all) {
 236                   $testline .= img_for($testfiles{'out'},1);
 237                 }
 238               } elsif ($testres->{'result'} eq "FAIL") {
 239                 if ($config_show_fail || $config_show_all) {
 240                   $testline .= img_for($testfiles{'out'},1);
 241                   $testline .= " ";
 242                   $testline .= img_for($testfiles{'diff'},1);
 243                   $testline .= " ";
 244                   $testline .= img_for($testfiles{'ref'},1);
 245                 }
 246               } elsif ($testres->{'result'} eq "CRASHED") {
 247                  $testline .= "!!!CRASHED!!!";
 248               } elsif ($testres->{'result'} eq "XFAIL") {
 249                 #nothing
 250                 if ($config_show_all) {
 251                   $testline .= img_for($testfiles{'out'},1);
 252                   #$testline .= "<hr size=\"1\">";
 253                   $testline .= " ";
 254                   $testline .= img_for($testfiles{'diff'},1);
 255                   $testline .= " ";
 256                   $testline .= img_for($testfiles{'ref'},1);
 257                 }
 258               } elsif ($testres->{'result'} eq "UNTESTED") {
 259                 #nothing
 260               } else {
 261                 $testline .= "UNSUPPORTED STATUS '$testres->{'result'}' (update make-html.pl)";
 262               }
 263
 264               $testline .= "</td>";
 265             } else {
 266               $testline .= '<td></td>';
 267             }
 268           } else {
 269             $testline .= '<td></td>';
 270           }
 271         }
 272         print '<tr><td>';
 273
 274         if ($config_show_inline) {
 275           print "$test ($format/$offset) ";
 276           print "(<a href=\"" . string_to_data("text/plain",$logs->{$test}) . "\">log</a>)";
 277         } else {
 278           print $test, ' (', $format, '/', $offset, ($similar ? ' similar' : ''), ') ';
 279           print "(<a href=\"$test.log\">log</a>)";
 280         }
 281
 282         print '</td>';
 283
 284         print $testline;
 285
 286         print "</tr>\n";
 287       }
 288     }
 289   }
 290 }
 291
 292 print "</table></body></html>\n";