bin/lo-commit-stat

   1 #!/usr/bin/perl
   2     eval 'exec /usr/bin/perl -S $0 ${1+"$@"}'
   3         if $running_under_some_shell;
   4 #!/usr/bin/perl
   5
   6 use strict;
   7 use LWP::UserAgent;
   8 use utf8;
   9 use File::Temp;
  10
  11 my %module_dirname = (
  12     "core"              => "",
  13     "dictionaries"      => "dictionaries",
  14     "help"              => "helpcontent2",
  15     "translations"      => "translations"
  16 );
  17
  18
  19 my %bugzillas = (
  20     fdo  => "https://bugs.documentfoundation.org/show_bug.cgi?id=",
  21     tdf  => "https://bugs.documentfoundation.org/show_bug.cgi?id=",
  22     bnc  => "https://bugzilla.novell.com/show_bug.cgi?id=",
  23     rhbz => "https://bugzilla.redhat.com/show_bug.cgi?id=",
  24     i    => "https://bz.apache.org/ooo/show_bug.cgi?id=",
  25     fate => "https://features.opensuse.org/",
  26 );
  27
  28 sub search_bugs($$$$)
  29 {
  30     my ($pdata, $module, $commit_id, $line) = @_;
  31
  32     my $bug = "";
  33     my $bug_orig;
  34     while (defined $bug) {
  35
  36         # match fdo#123, rhz#123, i#123, #123
  37         # but match only bug number with >= 4 digits
  38         if ( $line =~ m/(\w+\#+\d{4,})/ ) {
  39             $bug_orig = $1;
  40             $bug = $1;
  41                 # default to issuezilla for the #123 variant
  42         # but match only bug number with >= 4 digits
  43         } elsif ( $line =~ m/(\#)(\d{4,})/ ) {
  44             $bug_orig = $1 . $2;
  45             $bug = "i#$2";
  46         # match #i123#
  47         } elsif ( $line =~ m/(\#i)(\d+)(\#)/ ) {
  48             $bug_orig = $1 . $2 . $3;
  49             $bug = "i#$2";
  50         } else {
  51             $bug = undef;
  52             next;
  53         }
  54
  55 #        print "   found $bug\n";
  56         # remove bug number from the comment; it will be added later a standardized way
  57         $bug_orig =~ s/\#/\\#/;
  58         $line =~ s/(,\s)*[Rr](elated|esolve[ds]):?\s*$bug_orig\s*//;
  59         $line =~ s/\s*-\s*$bug_orig\s*//;
  60         $line =~ s/\(?$bug_orig\)?\s*[:,-]?\s*//;
  61
  62         # bnc# is preferred over n# for novell bugs
  63         $bug =~ s/^n\#/bnc#/;
  64         # deb# is preferred over debian# for debian bugs
  65         $bug =~ s/^debian\#/deb#/;
  66         # easyhack# is sometimes used for fdo# - based easy hacks
  67         $bug =~ s/^easyhack\#/fdo#/;
  68         # someone mistyped fdo as fd0
  69         $bug =~ s/^fd0\#/fdo#/;
  70         # save the bug number
  71         %{$pdata->{$module}{$commit_id}{'bugs'}} = () if (! defined %{$pdata->{$module}{$commit_id}{'bugs'}});
  72         $pdata->{$module}{$commit_id}{'bugs'}{$bug} = 1;
  73     }
  74
  75     return $line;
  76 }
  77
  78 sub standardize_summary($)
  79 {
  80     my $line = shift;
  81
  82     $line =~ s/^\s*//;
  83     $line =~ s/\s*$//;
  84
  85     # lower first letter if the word contains only lowercase letter
  86     if ( $line =~ m/(^.[a-z]+\b)/ ) {
  87         $line =~ m/(^.)/;
  88         my $first_char = lc($1);
  89         $line =~ s/^./$first_char/;
  90     }
  91
  92     # FIXME: remove do at the end of line
  93     #        remove bug numbers
  94     return $line;
  95 }
  96
  97 sub generate_git_cherry_ids_log($$$$$)
  98 {
  99     my ($pdata, $repo_dir, $module, $branch_name, $git_args) = @_;
 100
 101     my $commit_ids_log;
 102     my $commit_ids_log_fh;
 103     $commit_ids_log_fh = File::Temp->new(TEMPLATE => 'lo-commit-stat-ids-XXXXXX',
 104                                          DIR => '/tmp',
 105                                          UNLINK => 0);
 106     $commit_ids_log = $commit_ids_log_fh->filename;
 107
 108     print STDERR "Filtering cherry-picked commits in the git repo: $module...\n";
 109
 110     my $cmd = "cd $repo_dir; git cherry $git_args";
 111     open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
 112
 113     while (my $line = <GIT>) {
 114
 115         # skip cherry-picked commits
 116         next if ( $line =~ m/^\-/ );
 117
 118         if ( $line =~ m/^\+ / ) {
 119             $line =~ s/^\+ //;
 120             print $commit_ids_log_fh $line;
 121         }
 122     }
 123
 124     close GIT;
 125     close $commit_ids_log_fh;
 126
 127     return $commit_ids_log;
 128 }
 129
 130 sub load_git_log($$$$$$$)
 131 {
 132     my ($pdata, $repo_dir, $module, $branch_name, $git_command, $git_cherry, $git_args) = @_;
 133
 134     my $cmd = "cd $repo_dir;";
 135     my $commit_ids_log;
 136
 137     if ($git_cherry) {
 138         $commit_ids_log = generate_git_cherry_ids_log($pdata, $repo_dir, $module, $branch_name, $git_args);
 139         $cmd .= " cat $commit_ids_log | xargs -n 1 $git_command -1";
 140     } else {
 141         $cmd .= " $git_command $git_args";
 142     }
 143
 144     my $commit_id;
 145     my $summary;
 146
 147     print STDERR "Analyzing log from the git repo: $module...\n";
 148
 149 #   FIXME: ./g pull move submodules in unnamed branches
 150 #    my $repo_branch_name = get_branch_name($repo_dir);
 151 #    if ( $branch_name ne $repo_branch_name ) {
 152 #        die "Error: mismatch of branches:\n" .
 153 #            "       main repo is on the branch: $branch_name\n" .
 154 #            "       $module repo is on the branch: $repo_branch_name\n";
 155 #    }
 156
 157     open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
 158     %{$pdata->{$module}} = ();
 159
 160     while (my $line = <GIT>) {
 161         chomp $line;
 162
 163         if ( $line =~ m/^commit ([0-9a-z]{20})/ ) {
 164             $commit_id = "$1";
 165             $summary=undef;
 166             %{$pdata->{$module}{"$commit_id"}} = ();
 167             next;
 168         }
 169
 170         if ( $line =~ /^Author:\s*([^\<]*)\<([^\>]*)>/ ) {
 171             # get rid of extra empty spaces;
 172             my $name = "$1";
 173             $name =~ s/\s+$//;
 174             die "Error: Author already defined for the commit {$commit_id}\n" if defined ($pdata->{$module}{$commit_id}{'author'});
 175             %{$pdata->{$module}{$commit_id}{'author'}} = ();
 176             $pdata->{$module}{$commit_id}{'author'}{'name'} = "$name";
 177             $pdata->{$module}{$commit_id}{'author'}{'email'} = "$2";
 178             next;
 179         }
 180
 181         if ( $line =~ /^Date:\s+/ ) {
 182             # ignore date line
 183             next;
 184         }
 185
 186         if ( $line =~ /^\s*$/ ) {
 187             # ignore empty line
 188             next;
 189         }
 190
 191         $line = search_bugs($pdata, $module, $commit_id, $line);
 192         # FIXME: need to be implemented
 193 #        search_keywords($pdata, $line);
 194
 195         unless (defined $pdata->{$module}{$commit_id}{'summary'}) {
 196             $summary = standardize_summary($line);
 197             $pdata->{$module}{$commit_id}{'summary'} = $summary;
 198         }
 199     }
 200
 201     close GIT;
 202     unlink $commit_ids_log if ($git_cherry);
 203 }
 204
 205 sub get_repo_name($)
 206 {
 207     my $repo_dir = shift;
 208
 209     open (GIT_CONFIG, "$repo_dir/.git/config") ||
 210         die "can't open \"$$repo_dir/.git/config\" for reading: $!\n";
 211
 212     while (my $line = <GIT_CONFIG>) {
 213         chomp $line;
 214
 215         if ( $line =~ /^\s*url\s*=\s*(\S+)$/ ) {
 216             my $repo_name = "$1";
 217             $repo_name = s/.*\///g;
 218             return "$repo_name";
 219         }
 220     }
 221     die "Error: can't find repo name in \"$$repo_dir/.git/config\"\n";
 222 }
 223
 224 sub load_data($$$$$$$)
 225 {
 226     my ($pdata, $top_dir, $p_module_dirname, $branch_name, $git_command, $git_cherry, $git_args) = @_;
 227
 228     foreach my $module (sort { $a cmp $b } keys %{$p_module_dirname}) {
 229         load_git_log($pdata, "$top_dir/$p_module_dirname->{$module}", $module, $branch_name, $git_command, $git_cherry, $git_args);
 230     }
 231 }
 232
 233 sub get_branch_name($)
 234 {
 235     my ($top_dir) = @_;
 236
 237     my $branch;
 238     my $cmd = "cd $top_dir && git branch";
 239
 240     open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
 241
 242     while (my $line = <GIT>) {
 243         chomp $line;
 244
 245         if ( $line =~ m/^\*\s*(\S+)/ ) {
 246             $branch = "$1";
 247         }
 248     }
 249
 250     close GIT;
 251
 252     die "Error: did not detect git branch name in $top_dir\n" unless defined ($branch);
 253
 254     return $branch;
 255 }
 256
 257 sub get_bug_list($$$)
 258 {
 259     my ($pdata, $pbugs, $check_bugzilla) = @_;
 260
 261     # associate bugs with their summaries and fixers
 262     foreach my $module ( keys %{$pdata}) {
 263         foreach my $id ( keys %{$pdata->{$module}}) {
 264             foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) {
 265                 %{$pbugs->{$bug}} = () if (! defined %{$pbugs->{$bug}});
 266                 my $author = $pdata->{$module}{$id}{'author'}{'name'};
 267                 my $summary = $pdata->{$module}{$id}{'summary'};
 268                 $pbugs->{$bug}{'summary'} = $summary;
 269                 $pbugs->{$bug}{'author'}{$author} = 1;
 270             }
 271         }
 272     }
 273
 274     # try to replace summaries with bug names from bugzilla
 275     if ($check_bugzilla) {
 276         print "Getting bug titles:\n";
 277         foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) {
 278             $pbugs->{$bug}{'summary'} = get_bug_name($bug, $pbugs->{$bug}{'summary'});
 279         }
 280     }
 281 }
 282
 283 sub open_log_file($$$$$$)
 284 {
 285     my ($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_;
 286
 287     my $logfilename = "$log_prefix-$branch_name-$log_suffix";
 288     $logfilename = "$log_dir/$logfilename" if (defined $log_dir);
 289     if ($wiki) {
 290         $logfilename .= ".wiki";
 291     } else {
 292         $logfilename .= ".log";
 293     }
 294
 295     if (-f $logfilename) {
 296         print "WARNING: The log file already exists: $logfilename\n";
 297         print "Do you want to overwrite it? (Y/n)?\n";
 298         my $answer = <STDIN>;
 299         chomp $answer;
 300         $answer = "y" unless ($answer);
 301         die "Please, rename the file or choose another log suffix\n" if ( lc($answer) ne "y" );
 302     }
 303
 304     my $log;
 305     open($log, '>', $logfilename) || die "Can't open \"$logfilename\" for writing: $!\n";
 306
 307     return $log;
 308 }
 309
 310 sub print_commit_summary($$$$$$)
 311 {
 312     my ($summary, $pmodule_title, $pbugs, $pauthors, $prefix, $log) = @_;
 313
 314     return if ( $summary eq "" );
 315
 316     # print module title if not done yet
 317     if ( defined ${$pmodule_title} ) {
 318         print $log "${$pmodule_title}\n";
 319         ${$pmodule_title} = undef;
 320     }
 321
 322     # finally print the summary line
 323     my $bugs = "";
 324     if ( %{$pbugs} ) {
 325         $bugs = " (" . join (", ", keys %{$pbugs}) . ")";
 326     }
 327
 328     my $authors = "";
 329     if ( %{$pauthors} ) {
 330         $authors = " [" . join (", ", keys %{$pauthors}) . "]";
 331     }
 332
 333     print $log $prefix, $summary, $bugs, $authors, "\n";
 334 }
 335
 336 sub print_commits($$$)
 337 {
 338     my ($pdata, $log, $wiki) = @_;
 339
 340     foreach my $module ( sort { $a cmp $b } keys %{$pdata}) {
 341         # check if this module has any entries at all
 342         my $module_title = "+ $module";
 343         if ( %{$pdata->{$module}} ) {
 344             my $old_summary="";
 345             my %authors = ();
 346             my %bugs = ();
 347             foreach my $id ( sort { lc $pdata->{$module}{$a}{'summary'} cmp lc $pdata->{$module}{$b}{'summary'} } keys %{$pdata->{$module}}) {
 348                 my $summary = $pdata->{$module}{$id}{'summary'};
 349                 if ($summary ne $old_summary) {
 350                     print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, "    + ", $log);
 351                     $old_summary = $summary;
 352                     %authors = ();
 353                     %bugs = ();
 354                 }
 355                 # collect bug numbers
 356                 if (defined $pdata->{$module}{$id}{'bugs'}) {
 357                     foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) {
 358                         $bugs{$bug} = 1;
 359                     }
 360                 }
 361                 # collect author names
 362                 my $author = $pdata->{$module}{$id}{'author'}{'name'};
 363                 $authors{$author} = 1;
 364             }
 365             print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, "    + ", $log);
 366         }
 367     }
 368 }
 369
 370 sub get_bug_name($$)
 371 {
 372     my ($bug, $summary) = @_;
 373     print "$bug: ";
 374
 375     $bug =~ m/(?:(\w*)\#+(\d+))/; # fdo#12345
 376     my $bugzilla = $1;            # fdo
 377     my $bug_number = $2;          # 12345
 378
 379     if ( $bugzillas{$bugzilla} ) {
 380         my $url = $bugzillas{$bugzilla} . $bug_number;
 381         my $ua = LWP::UserAgent->new;
 382         $ua->timeout(10);
 383         $ua->env_proxy;
 384         my $response = $ua->get($url);
 385         if ($response->is_success) {
 386             my $title = $response->title;
 387             if ( $title =~ s/^Bug \d+ \S+ // ) {
 388                 print "$title\n";
 389                 return $title;
 390             } else {
 391                 print "warning: not found; using commit message (only got $title)\n";
 392             }
 393         } else {
 394             print "\n";
 395         }
 396     } else {
 397         print "\n";
 398     }
 399
 400     return $summary;
 401 }
 402
 403 sub print_bugs($$$$)
 404 {
 405     my ($pbugs, $log, $wiki) = @_;
 406
 407     foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) {
 408         my $summary = $pbugs->{$bug}{'summary'};
 409
 410         my $authors = "";
 411         if ( %{$pbugs->{$bug}{'author'}} ) {
 412             $authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]";
 413         }
 414
 415         $bug =~ s/(.*)\#(.*)/# {{$1|$2}}/ if ($wiki);
 416         print $log $bug, " ", $summary, $authors, "\n";
 417     }
 418 }
 419
 420 sub print_bugs_changelog($$$$)
 421 {
 422     my ($pbugs, $log, $wiki) = @_;
 423
 424     foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) {
 425         my $summary = $pbugs->{$bug}{'summary'};
 426
 427         my $authors = "";
 428         if ( %{$pbugs->{$bug}{'author'}} ) {
 429             $authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]";
 430         }
 431
 432         print $log "    + $summary ($bug)$authors\n";
 433     }
 434 }
 435
 436 sub print_bugnumbers($$$$)
 437 {
 438     my ($pbugs, $log, $wiki) = @_;
 439
 440     print $log join ("\n", sort { $a cmp $b } keys %{$pbugs}), "\n";
 441 }
 442
 443 sub generate_log($$$$$$$$)
 444 {
 445     my ($pused_data, $print_func, $log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_;
 446
 447     my $log = open_log_file($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki);
 448     & {$print_func} ($pused_data, $log, $wiki);
 449     close $log;
 450 }
 451
 452 ########################################################################
 453 # help
 454
 455 sub usage()
 456 {
 457     print "This script generates LO git commit summary\n\n" .
 458
 459           "Usage: lo-commit-stat [--help] [--no-submodules] [--module=<module>] --log-dir=<dir> --log-suffix=<string> topdir [git_arg...]\n\n" .
 460
 461           "Options:\n" .
 462           "     --help           print this help\n" .
 463           "     --no-submodule   read changes just from the main repository, ignore submodules\n" .
 464           "     --module=<module> summarize just changes from the given module, use \"core\"\n" .
 465           "                      for the main module\n" .
 466           "     --log-dir=<dir>  directory where to put the generated log\n" .
 467           "     --log-suffix=<string> suffix of the log file name; the result will be\n" .
 468           "                      commit-log-<branch>-<log-name-suffix>.log; the branch name\n" .
 469           "                      is detected automatically\n" .
 470           "     --commits        generete log with all commits (default)\n" .
 471           "     --bugs           generate log with bugzilla entries\n" .
 472           "     --bugs-changelog generate log with bugzilla entries, use changelog style\n" .
 473           "     --bugs-wiki      generate log with bugzilla entries, use wiki markup\n" .
 474           "     --bugs-numbers   generate log with bugzilla numbers\n" .
 475           "     --rev-list       use \"git rev-list\" instead of \"git log\"; useful to check\n" .
 476           "                      differences between branches\n" .
 477           "     --cherry         use \"git cherry\" instead of \"git log\"; detects cherry-picked\n" .
 478           "                      commits between branches\n" .
 479           "      topdir          directory with the libreoffice/core clone\n" .
 480           "      git_arg         extra parameters passed to the git command to define\n" .
 481           "                      the area of interest; The default command is \"git log\" and\n" .
 482           "                      parameters might be, for example, --after=\"2010-09-27\" or\n" .
 483           "                      TAG..HEAD; with the option --rev-list, useful might be, for\n" .
 484           "                      example origin/master ^origin/libreoffice-3-3; with the option\n" .
 485           "                      --rev-list, useful might be, for example libreoffice-3.6.3.2\n" .
 486           "                      libreoffice-3.6.4.1\n";
 487 }
 488
 489
 490 #######################################################################
 491 #######################################################################
 492 # MAIN
 493 #######################################################################
 494 #######################################################################
 495
 496
 497 my $module;
 498 my %generate_log = ();
 499 my $top_dir;
 500 my $log_dir;
 501 my $log_suffix;
 502 my $log;
 503 my $list_bugs = 0;
 504 my $check_bugzilla = 0;
 505 my $branch_name;
 506 my $git_command = "git log";
 507 my $git_cherry;
 508 my $git_args = "";
 509 my $branch_name;
 510 my %data;
 511 my %bugs = ();
 512
 513
 514 foreach my $arg (@ARGV) {
 515     if ($arg eq '--help') {
 516         usage();
 517         exit;
 518     } elsif ($arg eq '--no-submodule') {
 519         $module = "core";
 520     } elsif ($arg =~ m/--module=(.*)/) {
 521         $module = $1;
 522     } elsif ($arg =~ m/--log-suffix=(.*)/) {
 523         $log_suffix = "$1";
 524     } elsif ($arg =~ m/--log-dir=(.*)/) {
 525         $log_dir = "$1";
 526     } elsif ($arg eq '--commits') {
 527         $generate_log{"commits"} = 1;
 528     } elsif ($arg eq '--bugs') {
 529         $generate_log{"bugs"} = 1;
 530         $check_bugzilla = 1;
 531         $list_bugs = 1;
 532     } elsif ($arg eq '--bugs-changelog') {
 533         $generate_log{"bugs-changelog"} = 1;
 534         $check_bugzilla = 1;
 535         $list_bugs = 1;
 536     } elsif ($arg eq '--bugs-wiki' || $arg eq '--wikibugs') {
 537         $generate_log{"bugs-wiki"} = 1;
 538         $check_bugzilla = 1;
 539         $list_bugs = 1;
 540     } elsif ($arg eq '--bugs-numbers' || $arg eq '--bug-numbers') {
 541         $generate_log{"bugs-numbers"} = 1;
 542         $list_bugs = 1;
 543     } elsif ($arg eq '--rev-list') {
 544         $git_command = "git rev-list --pretty=medium"
 545     } elsif ($arg eq '--cherry') {
 546         $git_command = "git log";
 547         $git_cherry = 1;
 548     } else {
 549         if (! defined $top_dir) {
 550             $top_dir=$arg;
 551         } else {
 552             $git_args .= " $arg";
 553         }
 554     }
 555 }
 556
 557 # default log
 558 if (%generate_log == 0) {
 559     $generate_log{"commits"} = 1;
 560 }
 561
 562 # we want only one module
 563 if ($module) {
 564     my $name = $module_dirname{$module};
 565     %module_dirname = ();
 566     $module_dirname{$module} = $name;
 567 }
 568
 569 (defined $top_dir) || die "Error: top directory is not defined\n";
 570 (-d "$top_dir") || die "Error: not a directory: $top_dir\n";
 571 (-f "$top_dir/.git/config") || die "Error: can't find $top_dir/.git/config\n";
 572
 573 (!defined $log_dir) || (-d $log_dir) || die "Error: directory does no exist: $log_dir\n";
 574
 575 (defined $log_suffix) || die "Error: define log suffix using --log-suffix=<string>\n";
 576
 577 $branch_name = get_branch_name($top_dir);
 578
 579 load_data(\%data, $top_dir, \%module_dirname, $branch_name, $git_command, $git_cherry, $git_args);
 580 get_bug_list(\%data, \%bugs, $check_bugzilla) if ($list_bugs);
 581
 582 generate_log(\%data, \&print_commits,        $log_dir, "commits",        $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"commits"});
 583 generate_log(\%bugs, \&print_bugs,           $log_dir, "bugs",           $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs"});
 584 generate_log(\%bugs, \&print_bugs,           $log_dir, "bugs",           $log_suffix, $top_dir, $branch_name, 1) if (defined $generate_log{"bugs-wiki"});
 585 generate_log(\%bugs, \&print_bugs_changelog, $log_dir, "bugs-changelog", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-changelog"});
 586 generate_log(\%bugs, \&print_bugnumbers,     $log_dir, "bug-numbers",    $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-numbers"});