tools/automated_test_reports/generate_report.pl

   1 #!/usr/bin/env perl
   2
   3 use strict;
   4 use warnings;
   5 use English;
   6 use XML::Feed;
   7 use XML::Simple;
   8 use Config;
   9
  10 # Define your gmake. Needed for Unixes which have both gmake and make installed.
  11 # On Linux it could be named just 'make'.
  12 my $MAKE = 'make';
  13
  14 # define your path here if run_test.pl is not in path
  15 # e.g.
  16 # my $run_test = '/Users/klink/dev/openxpki/tools/automated_test_reports/run_test.pl';
  17 my $run_test;
  18
  19 # define an installation prefix (passed to perl Makefile.PL) if necessary
  20 # e.g.
  21 # my $PREFIX = 'PREFIX=~/usr/local';
  22 my $PREFIX = '';
  23
  24 # add installation prefix to PERL5LIB
  25 my $PREFIX_DIR = $PREFIX;
  26 $PREFIX_DIR =~ s{ \A \s* PREFIX=}{}xms if ($PREFIX ne '');
  27
  28 if ($PREFIX_DIR ne '') {
  29     my $installstyle = $Config{installstyle};
  30     my $perlversion = $Config{version};
  31     my $archname = $Config{archname};
  32     $ENV{"PERL5LIB"} = $PREFIX_DIR . "/$installstyle/site_perl/$perlversion/:" .
  33                        $PREFIX_DIR . "/$installstyle/site_perl/$perlversion/$archname/:" .
  34                        $PREFIX_DIR . "/$installstyle/site_perl/$perlversion/mach/";
  35     print STDERR "PERL5LIB contains the following dirs: ".$ENV{"PERL5LIB"}."\n";
  36 }
  37
  38 # define a deployment prefix (passed to ./configure in trunk/deployment)
  39 # if necessary
  40 # e.g.
  41 # my $DEPLOYMENT_PREFIX = '--prefix ~/usr/local';
  42 my $DEPLOYMENT_PREFIX = '';
  43
  44 # add deployment prefix to PATH
  45 # define DEPLOYMENT_PREFIX variable (to be used in OpenXPKI/Tests.pm)
  46 my $DEPLOYMENT_DIR = $DEPLOYMENT_PREFIX;
  47 $DEPLOYMENT_DIR =~ s{ \A \s* --prefix \s* (\S+)}{$1}xms if ($DEPLOYMENT_PREFIX ne '');
  48
  49 if ($DEPLOYMENT_DIR ne '') {
  50     $ENV{"PATH"} = $DEPLOYMENT_DIR."/bin:" . $ENV{"PATH"};
  51     $ENV{"DEPLOYMENT_PREFIX"} = $DEPLOYMENT_DIR."/bin";
  52 }
  53
  54 # define the format of the feed (see XML::Feed)
  55 my $FEED_FORMAT = 'Atom';
  56
  57 # define the title and location of the feed
  58 my $FEED_TITLE = "OpenXPKI tests at " . `hostname`;
  59 my $FEED_LINK  = "http://build0.cynops.de/openxpki_tests/";
  60
  61 # define how many revisions you want to show at the HTML page and how
  62 # many you want in the Atom feed
  63 my $MAX_HTML_REVISIONS = 20;
  64 my $MAX_ATOM_REVISIONS = 20;
  65
  66 if (! defined $run_test) {
  67     $run_test = `which run_test.pl`;
  68     chomp($run_test);
  69     if ($CHILD_ERROR) {
  70         die "Could not find run_test.pl, configure it in generate_report.pl or put it in your PATH";
  71     }
  72 }
  73
  74 if (! -e 'perl-modules/core/trunk/t') {
  75     # only run from the correct directory
  76     die "Please start from main OpenXPKI trunk directory\n";
  77 }
  78 my $basedir = `pwd`;
  79 chomp $basedir;
  80
  81 my $output_dir = $ARGV[0];
  82 if (! defined $output_dir) {
  83     die "Usage: $0 <output directory>";
  84 }
  85
  86 # Numbers added to the test names to provide correct sequence of installation:
  87 # server, client, other clients. Needed for testing on machines with no
  88 # pre-installed OpenXPKI.
  89
  90 my $tests = {
  91     '0_server'       => {
  92         DIRECTORY => $basedir . "/perl-modules/core/trunk",
  93         NAME      => 'Server',
  94     },
  95     '1_client'       => {
  96         DIRECTORY => $basedir . "/clients/perl/OpenXPKI-Client",
  97         NAME      => 'Client',
  98     },
  99     '2_client_mason' => {
 100         DIRECTORY => $basedir . "/clients/perl/OpenXPKI-Client-HTML-Mason",
 101         NAME      => 'Mason client',
 102     },
 103     '3_client_scep'  => {
 104         DIRECTORY => $basedir . "/clients/perl/OpenXPKI-Client-SCEP",
 105         NAME      => 'SCEP client',
 106     },
 107 };
 108
 109 my $revision = sprintf("%04d", `vergen --format SVN_REVISION`);
 110 if (! $revision) {
 111     die "Could not determine SVN revision";
 112 }
 113 if (! -x $run_test) {
 114     die "run_test.pl not found or not executable";
 115 }
 116
 117 # will contain test name without a number
 118 my $test;
 119
 120 INSTALL:
 121 foreach my $testname (sort keys %{ $tests }) {
 122     $test = $testname;
 123     $test =~ s{\d_}{}xms;
 124
 125     print STDERR "Compiling and installing for $test\n";
 126     chdir $tests->{$testname}->{DIRECTORY};
 127     if (system("perl Makefile.PL $PREFIX && $MAKE && $MAKE install") != 0) {
 128         die "Could not compile for $test test";
 129     }
 130 }
 131
 132 # install new deployment tools before testing
 133 chdir $basedir . '/deployment';
 134 if (system("./configure $DEPLOYMENT_PREFIX && $MAKE && $MAKE install") != 0) {
 135     die "Could not install new deployment tools";
 136 }
 137
 138 TEST:
 139 foreach my $testname (sort keys %{ $tests }) {
 140     $test = $testname;
 141     $test =~ s{\d_}{}xms;
 142
 143     print STDERR "Running test for $test (revision $revision)\n";
 144     chdir $tests->{$testname}->{DIRECTORY};
 145
 146     # compile and run each test
 147     my $output_filename = $revision . '_output.txt';
 148     if (-e $output_filename) {
 149         # test has already been run, skip
 150         next TEST;
 151     }
 152     my $ENV = '';
 153     if ($test eq 'client_mason') {
 154         # we need to set this here, because any attempt to set it within
 155         # the test file itself fails for some reason ...
 156         $ENV = 'OPENXPKI_SOCKET_FILE=t/20_webserver/test_instance/var/openxpki/openxpki.socket';
 157     }
 158     system("$ENV $run_test $revision $test");
 159 }
 160
 161 chdir $tests->{'0_server'}->{DIRECTORY};
 162 my @tested_revisions = reverse glob('*_output.txt');
 163 map { s/_output.txt// } @tested_revisions;
 164
 165 open INDEX, '>', $output_dir . '/index.html';
 166 print INDEX << "XEOF";
 167 <?xml version="1.0" encoding="utf-8"?>
 168 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 169 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
 170   <head>
 171     <title>OpenXPKI automated test report</title>
 172   </head>
 173   <body>
 174     <table border=0 cellspacing=0 cellpadding=5>
 175       <tr>
 176           <th>Coverage report</th>
 177           <th>Good files</th>
 178           <th>Bad files</th>
 179           <th>Total files</th>
 180           <th>Skipped files</th>
 181           <th>Good tests</th>
 182           <th>Bad tests</th>
 183           <th>Total tests</th>
 184           <th>TODO tests</th>
 185           <th>passed TODO tests</th>
 186           <th>skipped tests</th>
 187           <th>CPU time used</th>
 188       </tr>
 189 XEOF
 190
 191 print STDERR "Generating report ... ";
 192 REV_REPORT:
 193 foreach my $rev (@tested_revisions[0..$MAX_HTML_REVISIONS-1]) {
 194     next REV_REPORT if (!$rev);
 195     print STDERR "r$rev ";
 196     chdir $basedir;
 197     open my $SVN, "svn log -qr $rev ..|";
 198
 199     # my $line = <$SVN>;
 200     # my $svn_info = <$SVN>;
 201     ## execution of the two previous lines fails on FreeBSD
 202     ## with 'Broken pipe' error message
 203
 204     my $svn_info;
 205     my $i = 0;
 206     while (my $line = <$SVN>) {
 207         chomp $line;
 208         $svn_info = $line if ($i == 1);
 209         $i++;
 210     }
 211     close $SVN;
 212     $svn_info =~ s{ (.*) \|\ \d+\ lines}{$1}xms;
 213
 214     print INDEX "<tr><td colspan=12>Report for $svn_info</td></tr>";
 215     foreach my $testname (sort keys %{ $tests }) {
 216         $test = $testname;
 217         $test =~ s{\d_}{}xms;
 218
 219         chdir $tests->{$testname}->{DIRECTORY};
 220         my $files = $rev . '_coverage ' . $rev . '_output.txt ';
 221         system("mkdir $output_dir/$test 2>/dev/null");
 222
 223         ## 'cp -r' does not work on FreeBSD
 224         system("cp -R $files $output_dir/$test");
 225
 226         my $summary_filename = $rev . '_summary.html';
 227         my $opened = open my $SUMMARY, '<', $summary_filename;
 228         my $summary;
 229         if (! $opened) {
 230             $summary = '<tr color="303030"><td colspan=12>' . $tests->{$testname}->{NAME} . ' summary unavailable</td></tr>' . "\n";
 231         }
 232         else {
 233             $summary = <$SUMMARY>;
 234             $summary =~ s{Coverage\ report}{$tests->{$testname}->{NAME} coverage report}xms;
 235             close $SUMMARY;
 236         }
 237         print INDEX $summary;
 238     }
 239 }
 240 print STDERR "\n";
 241 close INDEX;
 242
 243 print STDERR "Generating feed ... ";
 244 my $feed = XML::Feed->new($FEED_FORMAT);
 245 $feed->title($FEED_TITLE);
 246 $feed->link($FEED_LINK);
 247
 248 REV_FEED:
 249 foreach my $rev (@tested_revisions[0..$MAX_ATOM_REVISIONS-1]) {
 250     next REV_FEED if (!$rev);
 251     my $entry = XML::Feed::Entry->new($FEED_FORMAT);
 252
 253     print STDERR "r$rev ";
 254     chdir $basedir;
 255     open my $SVN, "svn log -qr $rev ..|";
 256
 257     # my $line = <$SVN>;
 258     # my $svn_info = <$SVN>;
 259     ## execution of the two previous lines fails on FreeBSD
 260     ## with 'Broken pipe' error message
 261
 262     my $svn_info;
 263     my $i = 0;
 264     while (my $line = <$SVN>) {
 265         chomp $line;
 266         $svn_info = $line if ($i == 1);
 267         $i++;
 268     }
 269     close $SVN;
 270
 271     my @svn_info = split(/ \| /, $svn_info);
 272     my $author = $svn_info[1];
 273     my $date   = $svn_info[2];
 274     my ($year, $month, $day, $hour, $minute, $second, $tz) =
 275         ($date =~ m{\A (\d{4}) \- (\d{2}) \- (\d{2}) [ ]
 276                        (\d{2}) : (\d{2}) : (\d{2}) [ ] ([\+\-]\d{4})}xms);
 277     my $dt = DateTime->new(
 278         year   => $year,
 279         month  => $month,
 280         day    => $day,
 281         hour   => $hour,
 282         minute => $minute,
 283         second => $second,
 284         time_zone => $tz,
 285     );
 286
 287     my $svn_changes = do {
 288         local $INPUT_RECORD_SEPARATOR;
 289         open my $SVN, "svn log -vr $rev ..|";
 290         <$SVN>;
 291     };
 292
 293     my $all_green = 1;
 294     my $all_summaries = '<table border=0 cellspacing=0 cellpadding=5>';
 295     my @failed;
 296     foreach my $testname (sort keys %{ $tests }) {
 297         $test = $testname;
 298         $test =~ s{\d_}{}xms;
 299
 300         chdir $tests->{$testname}->{DIRECTORY};
 301         my $summary_filename = $rev . '_summary.html';
 302         my $opened = open my $SUMMARY, '<', $summary_filename;
 303         if (! $opened) {
 304             $all_green = 0;
 305             push @failed, "$test n/a";
 306         }
 307         else {
 308             my $summary = <$SUMMARY>;
 309             $summary =~ s{Coverage\ report}{$tests->{$testname}->{NAME} coverage report}xms;
 310             my $xs = XML::Simple->new();
 311             my $summary_ref = $xs->XMLin($summary);
 312             if ($summary_ref->{'td'}->[6] ne '0 (0.00%)') {
 313                 $all_green = 0;
 314                 # push failed total tests
 315                 push @failed, "$test: " . $summary_ref->{'td'}->[6];
 316             }
 317             $all_summaries .= $summary;
 318             close $SUMMARY;
 319         }
 320     }
 321     $all_summaries .= '</table>';
 322     if ($all_green) {
 323         $entry->title("$rev: All green");
 324     }
 325     else {
 326         $entry->title("$rev: Failed: " . join q{, }, @failed);
 327     }
 328     $entry->author($author);
 329     my $body = "<H1>Report for $rev</H1>" . $all_summaries;
 330     $body   .= "<H1>Changes</H1><pre>" . $svn_changes . "</pre>";
 331     $entry->content(XML::Feed::Content->new({body => $body }));
 332     $entry->modified($dt);
 333     $entry->link($FEED_LINK);
 334
 335     $feed->add_entry($entry);
 336 }
 337 $feed->modified(DateTime->now);
 338 open FEED, '>', "$output_dir/" . lc($FEED_FORMAT);
 339 print FEED $feed->as_xml();
 340 close(FEED);
 341 print STDERR "\n";