git-update-dirs

   1 #!/usr/bin/env perl
   2
   3 #==============================================================================
   4 # git-update-dirs
   5 # File ID: f1ba77e4-444e-11e0-963c-00023faf1383
   6 #
   7 # Update many git local repositories at once
   8 #
   9 # Character set: UTF-8
  10 # ©opyleft 2011– Øyvind A. Holm <sunny@sunbase.org>
  11 # License: GNU General Public License version 2 or later, see end of file for
  12 # legal stuff.
  13 #==============================================================================
  14
  15 use strict;
  16 use warnings;
  17 use Cwd qw{ abs_path getcwd };
  18 use Getopt::Long;
  19 use File::Basename;
  20
  21 local $| = 1;
  22
  23 our %Opt = (
  24
  25         'aggressive-compress' => 0,
  26         'all-options' => 0,
  27         'allbr' => 0,
  28         'compress' => 0,
  29         'dangling' => 0,
  30         'delete-dangling' => 0,
  31         'dry-run' => 0,
  32         'fetch' => 0,
  33         'fetch-prune' => 0,
  34         'ga-dropget' => 0,
  35         'ga-dropunused' => 0,
  36         'ga-getnew' => 0,
  37         'ga-moveunused' => 0,
  38         'ga-sync' => 0,
  39         'ga-update-desc' => 0,
  40         'help' => 0,
  41         'lpar' => 0,
  42         'pull' => 0,
  43         'push' => 0,
  44         'quiet' => 0,
  45         'recursive' => 0,
  46         'submodule' => 0,
  47         'test' => 0,
  48         'verbose' => 0,
  49         'version' => 0,
  50
  51 );
  52 our @opt_dirs_from = ();
  53 our @opt_exec_after = ();
  54 our @opt_exec_before = ();
  55
  56 our $progname = $0;
  57 $progname =~ s/^.*\/(.*?)$/$1/;
  58 our $VERSION = '0.6.6';
  59
  60 my $archive_disk = "seagate-3tb"; # FIXME: Hardcoding
  61
  62 Getopt::Long::Configure('bundling');
  63 GetOptions(
  64
  65         'aggressive-compress|C' => \$Opt{'aggressive-compress'},
  66         'all-options|A' => \$Opt{'all-options'},
  67         'allbr|a+' => \$Opt{'allbr'},
  68         'compress|c' => \$Opt{'compress'},
  69         'dangling|d' => \$Opt{'dangling'},
  70         'delete-dangling|D' => \$Opt{'delete-dangling'},
  71         'dirs-from=s' => \@opt_dirs_from,
  72         'dry-run|n' => \$Opt{'dry-run'},
  73         'exec-after|e=s' => \@opt_exec_after,
  74         'exec-before|E=s' => \@opt_exec_before,
  75         'fetch-prune|F' => \$Opt{'fetch-prune'},
  76         'fetch|f' => \$Opt{'fetch'},
  77         'ga-dropget|G' => \$Opt{'ga-dropget'},
  78         'ga-dropunused|u' => \$Opt{'ga-dropunused'},
  79         'ga-getnew|N' => \$Opt{'ga-getnew'},
  80         'ga-moveunused|U' => \$Opt{'ga-moveunused'},
  81         'ga-sync|g' => \$Opt{'ga-sync'},
  82         'ga-update-desc|S' => \$Opt{'ga-update-desc'},
  83         'help|h' => \$Opt{'help'},
  84         'lpar|l' => \$Opt{'lpar'},
  85         'pull|p' => \$Opt{'pull'},
  86         'push|P' => \$Opt{'push'},
  87         'quiet|q+' => \$Opt{'quiet'},
  88         'recursive|r' => \$Opt{'recursive'},
  89         'submodule|s' => \$Opt{'submodule'},
  90         'test|t' => \$Opt{'test'},
  91         'verbose|v+' => \$Opt{'verbose'},
  92         'version' => \$Opt{'version'},
  93
  94 ) || die("$progname: Option error. Use -h for help.\n");
  95
  96 $Opt{'verbose'} -= $Opt{'quiet'};
  97 $Opt{'help'} && usage(0);
  98 if ($Opt{'version'}) {
  99         print_version();
 100         exit(0);
 101 }
 102
 103 exit(main());
 104
 105 sub main {
 106         # {{{
 107         my $Retval = 0;
 108
 109         ($Opt{'ga-dropunused'} && $Opt{'ga-moveunused'}) &&
 110                 die("$progname: Can't use -u/--ga-dropunused and " .
 111                     "-U/--ga-moveunused at the same time\n");
 112
 113         if ($Opt{'all-options'}) {
 114                 $Opt{'allbr'} += 1;
 115                 $Opt{'dangling'} = 1;
 116                 $Opt{'fetch-prune'} = 1;
 117                 $Opt{'ga-sync'} = 1;
 118                 $Opt{'ga-update-desc'} = 1;
 119                 $Opt{'lpar'} = 1;
 120                 $Opt{'pull'} = 1;
 121                 $Opt{'push'} = 1;
 122                 $Opt{'submodule'} = 1;
 123         }
 124         ($Opt{'ga-dropget'} || $Opt{'ga-dropunused'}
 125             || $Opt{'ga-moveunused'}) &&
 126                 ($Opt{'ga-sync'} = 1);
 127
 128         my @Dirs = @ARGV;
 129         my @err_compress = ();
 130         my @err_dangling = ();
 131         my @err_fetch = ();
 132         my @err_gasync = ();
 133         my @err_pull = ();
 134         my @err_push = ();
 135         my @err_test = ();
 136         my @err_updsub = ();
 137
 138         if ($Opt{'recursive'}) {
 139                 my $repos = `find . -type d -name .git -print0`;
 140                 $repos =~ s/\/\.git\000/\000/g;
 141                 my %dupdir;
 142                 @Dirs = sort(grep { !$dupdir{$_}++ }
 143                         (@Dirs, split("\000", $repos)));
 144         }
 145
 146         my $orig_dir = getcwd();
 147         my ($total_before, $total_after, $total_saved) = (0, 0, 0);
 148         my ($totnum_before, $totnum_after) = (0, 0);
 149
 150         for my $dirfile (@opt_dirs_from) {
 151                 if ($dirfile eq '-') {
 152                         while (<STDIN>) {
 153                                 chomp();
 154                                 push(@Dirs, $_);
 155                         }
 156                 } else {
 157                         open(DirFP, "<$dirfile") or
 158                                 die("$progname: $dirfile: Cannot read " .
 159                                     "from file: $!\n");
 160                         while (<DirFP>) {
 161                                 chomp();
 162                                 push(@Dirs, $_);
 163                         }
 164                         close(DirFP);
 165                 }
 166         }
 167
 168         LOOP: for my $f (@Dirs) {
 169                 my $object_dir = '';
 170                 -d "$f/.git/." && ($object_dir = ".git/objects");
 171                 -d "$f/objects/." && ($object_dir = "objects");
 172                 msg(2, "object_dir = '$object_dir'");
 173                 if (length($object_dir)) {
 174                         if (!chdir($f)) {
 175                                 warn("$progname: $f: Cannot chdir: $!\n");
 176                                 next LOOP;
 177                         }
 178                         $Opt{'verbose'} >= -1 &&
 179                                 printf("================ %s "
 180                                        . "================\n",
 181                                        abs_path(getcwd()));
 182                         inside_git_dir() &&
 183                                 (chdir('..'), $object_dir = ".git/objects");
 184                         my $is_bare = (get_config('core.bare') =~ /^true/)
 185                                           ? 1
 186                                           : 0;
 187                         msg(2, "is_bare = '$is_bare'");
 188
 189                         my $Lh = "[0-9a-fA-F]";
 190                         my $uuid_templ = "$Lh\{8}-$Lh\{4}-$Lh\{4}-" .
 191                                          "$Lh\{4}-$Lh\{12}";
 192                         my $is_annex = (get_config('annex.uuid') =~
 193                                        /^$uuid_templ/)
 194                                            ? 1
 195                                            : 0;
 196                         msg(2, "is_annex = '$is_annex'");
 197
 198                         if (should('exec-before')) {
 199                                 for my $arg (@opt_exec_before) {
 200                                         mysystem($arg);
 201                                 }
 202                         }
 203                         if (should('lpar')) {
 204                                 mysystem("lpar");
 205                         }
 206                         if (should('test')) {
 207                                 mysystem("git", "fsck") && (
 208                                         push(@err_test, $f),
 209                                         warn("$progname: $f: ERRORS FOUND! " .
 210                                              "Skipping other actions " .
 211                                              "for this repo.\n"),
 212                                         next LOOP
 213                                 );
 214                         }
 215                         if (should('fetch-prune')) {
 216                                 mysystem("git", "fetch", "--all", "--prune") &&
 217                                         push(@err_fetch, $f);
 218                         } elsif (should('fetch')) {
 219                                 mysystem("git", "fetch", "--all") &&
 220                                         push(@err_fetch, $f);
 221                         }
 222                         if (should('pull') && !$is_bare) {
 223                                 mysystem("git", "pull", "--ff-only") &&
 224                                         push(@err_pull, $f);
 225                                 if (-e '.emptydirs') {
 226                                         mysystem('git', 'restore-dirs');
 227                                 }
 228                         }
 229                         if ($is_annex &&
 230                             (should('ga-sync') || should('ga-getnew'))) {
 231                                 should('ga-sync') && mysystem("ga", "sync") &&
 232                                         push(@err_gasync, $f);
 233                                 if (should('ga-dropget')) {
 234                                         mysystem("ga", "drop", "--auto");
 235                                         mysystem("ga", "sync");
 236                                 }
 237                                 if (should('ga-dropunused')) {
 238                                         mysystem("ga", "unused");
 239                                         mysystem("ga", "dropunused", "all");
 240                                         mysystem("ga", "sync");
 241                                 }
 242                                 if (should('ga-moveunused')) {
 243                                         if (open(my $fh, "git remote |")) {
 244                                                 my $found = 0;
 245                                                 while (my $curr_remote = <$fh>) {
 246                                                         if ($curr_remote =~ /^$archive_disk$/) {
 247                                                                 mysystem("ga", "unused");
 248                                                                 mysystem("ga", "move", "--unused",
 249                                                                          "--to", $archive_disk);
 250                                                                 mysystem("ga", "sync");
 251                                                                 last;
 252                                                         }
 253                                                 }
 254                                                 close($fh);
 255                                         } else {
 256                                                 warn("$progname: Cannot open " .
 257                                                      "'git remote' pipe: $!\n");
 258                                         }
 259                                 }
 260                                 if (should('ga-dropget') || should('ga-getnew')) {
 261                                         if (!$Opt{'ga-getnew'}) {
 262                                                 mysystem("ga", "get",
 263                                                          "--auto");
 264                                                 mysystem("ga", "sync");
 265                                         } else {
 266                                                 mysystem("ga-getnew");
 267                                         }
 268                                 }
 269                         }
 270                         if ($is_annex && should('ga-update-desc')) {
 271                                 mysystem("ga", "update-desc");
 272                         }
 273                         if (should('dangling')) {
 274                                 mysystem("git", "dangling") &&
 275                                         push(@err_dangling, $f);
 276                         }
 277                         if (should('allbr') &&
 278                             ($is_bare || $Opt{'allbr'} > 1)) {
 279                                 mysystem("git", "nobr");
 280                                 mysystem("git", "allbr", "-a");
 281                                 mysystem("git", "checkout", "-");
 282                         }
 283                         if (should('push')) {
 284                                 mysystem("git", "pa") && push(@err_push, $f);
 285                         }
 286                         if (should('submodule') && -e ".gitmodules") {
 287                                 mysystem("git", "submodule", "init");
 288                                 mysystem("git", "submodule", "update") &&
 289                                         push(@err_updsub, $f);
 290                         }
 291                         if (should('compress') ||
 292                             should('aggressive-compress')) {
 293                                 chomp(my $before =
 294                                     `(find $object_dir -type f -printf '%s+' ; echo 0) | bc`);
 295                                 $total_before += $before;
 296                                 chomp(my $numfiles_before =
 297                                     `find $object_dir -type f | wc -l`);
 298                                 $totnum_before += $numfiles_before;
 299                                 $Opt{'dry-run'} || print("\n");
 300                                 mysystem("git", "count-objects", "-vH");
 301                                 $Opt{'dry-run'} || print("\n");
 302                                 if (should('aggressive-compress')) {
 303                                         mysystem("git gc --aggressive") &&
 304                                                 push(@err_compress, $f);
 305                                 } else {
 306                                         mysystem("git gc") &&
 307                                                 push(@err_compress, $f);
 308                                 }
 309                                 chomp(my $after =
 310                                     `(find $object_dir -type f -printf '%s+' ; echo 0) | bc`);
 311                                 $total_after += $after;
 312                                 my $saved = $before - $after;
 313                                 chomp(my $numfiles_after =
 314                                       `find $object_dir -type f | wc -l`);
 315                                 $totnum_after += $numfiles_after;
 316                                 $before &&
 317                                         printf("\nBefore: %s\n" .
 318                                                "After : %s\n" .
 319                                                "Saved : %s (%.4f%%)\n",
 320                                                commify($before),
 321                                                commify($after),
 322                                                commify($saved),
 323                                                100.0 * $saved / $before);
 324                                 printf("Number of files in %s: " .
 325                                        "before: %u, after: %u, saved: %d\n",
 326                                        $object_dir,
 327                                        $numfiles_before,
 328                                        $numfiles_after,
 329                                        $numfiles_before-$numfiles_after)
 330                                 # Temporarily (?) disabled, it takes a heck of
 331                                 # a long time and uses loads of CPU and memory.
 332                                 # If the git-annex repo gets corrupted it's in
 333                                 # most cases good enough to delete
 334                                 # .git/annex/index anyway, and that can be done
 335                                 # manually.
 336                                 # $is_annex && should('dangling') && mysystem("ga", "repair");
 337                         }
 338                         if (should('delete-dangling') && !$is_bare) {
 339                                 mysystem("git", "dangling", "-D");
 340                         }
 341                         if (should('lpar')) {
 342                                 mysystem("lpar");
 343                         }
 344                         if (should('exec-after')) {
 345                                 for my $arg (@opt_exec_after) {
 346                                         mysystem($arg);
 347                                 }
 348                         }
 349                         $Opt{'verbose'} >= -1 && print("\n");
 350                 }
 351                 chdir($orig_dir) || die(
 352                     "$progname: $orig_dir: Cannot return to " .
 353                     "original directory: $!\n");
 354         }
 355         scalar(@err_fetch) && print("$progname: Unable to fetch from: " .
 356                                     join(" ", @err_fetch) . "\n");
 357         scalar(@err_gasync) && print("$progname: Unable to run ga sync: " .
 358                                      join(" ", @err_gasync) . "\n");
 359         scalar(@err_dangling) &&
 360                 print("$progname: Unable to run git dangling: " .
 361                       join(" ", @err_dangling) . "\n");
 362         scalar(@err_pull) && print("$progname: Unable to pull from: " .
 363                                    join(" ", @err_pull) . "\n");
 364         scalar(@err_push) && print("$progname: Unable to push from: " .
 365                                    join(" ", @err_push) . "\n");
 366         scalar(@err_updsub) &&
 367                 print("$progname: Unable to update submodules in: " .
 368                       join(" ", @err_updsub) . "\n");
 369         scalar(@err_compress) && print("$progname: Unable to compress: " .
 370                                        join(" ", @err_compress) . "\n");
 371         scalar(@err_test) && print("$progname: Error in git fsck: " .
 372                                    join(" ", @err_test) . "\n");
 373
 374         if ($Opt{'compress'} || $Opt{'aggressive-compress'}) {
 375                 my $total_saved = $total_before - $total_after;
 376                 printf("Before: %s\nAfter : %s\n",
 377                     commify($total_before), commify($total_after));
 378                 $total_before &&
 379                         printf("Total : %s (%.4f%%)\n",
 380                                commify($total_saved),
 381                                100.0 * $total_saved / $total_before);
 382                 printf("Number of object files: before: %u, after: %u, " .
 383                        "saved: %d\n", $totnum_before, $totnum_after,
 384                        $totnum_before-$totnum_after)
 385         }
 386
 387         return $Retval;
 388         # }}}
 389 }
 390
 391 sub check_sig {
 392         # {{{
 393         my $retval = shift;
 394         ($retval & 127) &&
 395                 die("\n$progname: Child process interrupted, aborting.\n");
 396         return(0);
 397         # }}}
 398 } # check_sig()
 399
 400 sub commify {
 401         # {{{
 402         my $Str = reverse $_[0];
 403         $Str =~ s/(\d\d\d)(?=\d)(?!\d*\,)/$1,/g;
 404         return scalar reverse $Str;
 405         # }}}
 406 } # commify()
 407
 408 sub get_config {
 409         # {{{
 410         my $name = shift;
 411         my $retval = '';
 412         chomp($retval = `git config --get "$name"`);
 413         return($retval);
 414         # }}}
 415 } # get_config()
 416
 417 sub inside_git_dir {
 418         # {{{
 419         my $basename = basename(abs_path(getcwd()));
 420         my $retval = ($basename eq '.git') ? 1 : 0;
 421         return($retval);
 422         # }}}
 423 } # inside_git_dir()
 424
 425 sub mysystem {
 426         # {{{
 427         my @cmd = @_;
 428         msg(0, sprintf("%s '", $Opt{'dry-run'} ? "Simulating" : "Executing") .
 429                                join(" ", @cmd) . "'...");
 430         $? = 0;
 431         !$Opt{'dry-run'} && system(@cmd) && check_sig($?);
 432         return $?;
 433         # }}}
 434 } # mysystem()
 435
 436 sub print_version {
 437         # Print program version {{{
 438         print("$progname $VERSION\n");
 439         return;
 440         # }}}
 441 }
 442
 443 sub should {
 444         # {{{
 445         my $name = shift;
 446         get_config("git-update-dirs.no-$name") eq 'true' && return 0;
 447         my $retval = 0;
 448         if ($name =~ /^exec-(before|after)$/) {
 449                 scalar($1 eq "before" ? @opt_exec_before : @opt_exec_after) &&
 450                         ($retval = 1);
 451         } else {
 452                 $Opt{$name} && ($retval = 1);
 453         }
 454         return($retval);
 455         # }}}
 456 } # should()
 457
 458 sub usage {
 459         # Send the help message to stdout {{{
 460         my $Retval = shift;
 461
 462         if ($Opt{'verbose'}) {
 463                 print("\n");
 464                 print_version();
 465         }
 466         print(<<"END");
 467
 468 Execute a predefined or customised set of commands in multiple local Git
 469 repositories in one operation.
 470
 471 Usage: $progname [options] [directories [...]]
 472
 473 Options, listed in the order they are executed in every Git repository:
 474
 475   -E X, --exec-before X
 476     Execute command X in every repo before all other commands. This
 477     option can be specified multiple times to run several commands.
 478     To disable: git config git-update-dirs.no-exec-before true
 479   -l, --lpar
 480     Execute lpar before and after fetch/pull and push
 481     To disable: git config git-update-dirs.no-lpar true
 482   -t, --test
 483     Test integrity of local repositories by running "git fsck".
 484     To disable: git config git-update-dirs.no-test true
 485   -F, --fetch-prune
 486     Fetch new commits from all remotes and prune deleted remote
 487     branches.
 488     To disable: git config git-update-dirs.no-fetch-prune true
 489   -f, --fetch
 490     Fetch new commits from all remotes.
 491     To disable: git config git-update-dirs.no-fetch true
 492   -p, --pull
 493     Also execute "git pull --ff-only".
 494     To disable: git config git-update-dirs.no-pull true
 495   -g, --ga-sync
 496     If the repo is used by git-annex, run "ga sync".
 497     To disable: git config git-update-dirs.no-ga-sync true
 498   -G, --ga-dropget
 499     Drop annex files having more copies than necessary, get files with
 500     fewer copies than necessary.
 501     To disable: git config git-update-dirs.no-ga-dropget true
 502   -u, --ga-dropunused
 503     In a git-annex repo, run "ga unused" followed by "ga dropunused
 504     all". Can't be used together with -U/--ga-moveunused.
 505     To disable: git config git-update-dirs.no-ga-dropunused true
 506   -U, --ga-moveunused
 507     Move unused git-annex contents to the '$archive_disk' remote. Can't
 508     be used together with -u/--ga-dropunused.
 509     To disable: git config git-update-dirs.no-ga-moveunused true
 510   -N, --ga-getnew
 511     Execute "ga-getnew", i.e. use "ga get --auto" to get all files from
 512     one month back that don't have enough copies.
 513     To disable: git config git-update-dirs.no-ga-getnew true
 514   -S, --ga-update-desc
 515     If the directory is controlled by git-annex, execute "ga
 516     update-desc" to set the description to the output from ga-pwd(1).
 517     To disable: git config git-update-dirs.no-ga-update-desc true
 518   -d, --dangling
 519     Execute "git dangling", i.e. turn all dangling commits into
 520     branches.
 521     To disable: git config git-update-dirs.no-dangling true
 522   -a, --allbr
 523     Execute "git nobr", "git allbr -a" and "git checkout -". If this
 524     option is specified once, it's only executed in bare repos. To also
 525     execute it in non-bare repos, it must be specified twice.
 526     To disable: git config git-update-dirs.no-allbr true
 527   -P, --push
 528     Also execute "git pa".
 529     To disable: git config git-update-dirs.no-push true
 530   -s, --submodule
 531     Update submodules if .gitmodules is found.
 532     To disable: git config git-update-dirs.no-submodule true
 533   -c, --compress
 534     Compress local repositories to save space.
 535     To disable: git config git-update-dirs.no-compress true
 536   -C, --aggressive-compress
 537     Use --aggressive when compressing the repository.
 538     To disable: git config git-update-dirs.no-aggressive-compress true
 539   -D, --delete-dangling
 540     Execute "git dangling -D" after execution to remove local commit-*
 541     branches and tag-* tags. This option is ignored in bare repos.
 542     To disable: git config git-update-dirs.no-delete-dangling true
 543   -e X, --exec-after X
 544     Execute command X in every repo after all other commands. This
 545     option can be specified multiple times to run several commands.
 546     To disable: git config git-update-dirs.no-exec-after true
 547
 548   -A, --all-options
 549     Run the program as if "-lFpgSdaPs" had been specified.
 550   --dirs-from X
 551     Read directory list from file X. If "-" is specified, read from
 552     stdin. Can be specified multiple times to read from different files.
 553   -h, --help
 554     Show this help.
 555   -n, --dry-run
 556     Simulate, don't actually execute any git commands.
 557   -q, --quiet
 558     Be more quiet. Can be repeated to increase silence.
 559   -r, --recursive
 560     Update all repositories recursively under the current directory.
 561   -v, --verbose
 562     Increase level of verbosity. Can be repeated.
 563   --version
 564     Print version information.
 565
 566 To disable some of these commands in a specific repository, set the git
 567 config variable git-update-dirs.no-OPTION to "true". For example, to
 568 disable push:
 569
 570   git config git-update-dirs.no-push true
 571
 572 Or disable aggressive compression:
 573
 574   git config git-update-dirs.no-aggressive-compress true
 575
 576 In this case aggressive compression will be disabled, and it will fall
 577 back to regular compression.
 578
 579 Only the value "true" (with lower case letters) is recognised, any other
 580 value will allow the command to run.
 581
 582 END
 583         exit($Retval);
 584         # }}}
 585 }
 586
 587 sub msg {
 588         # Print a status message to stderr based on verbosity level {{{
 589         my ($verbose_level, $Txt) = @_;
 590
 591         if ($Opt{'verbose'} >= $verbose_level) {
 592                 print(STDERR "$progname: $Txt\n");
 593         }
 594         return;
 595         # }}}
 596 }
 597
 598 __END__
 599
 600 # This program is free software; you can redistribute it and/or modify it under
 601 # the terms of the GNU General Public License as published by the Free Software
 602 # Foundation; either version 2 of the License, or (at your option) any later
 603 # version.
 604 #
 605 # This program is distributed in the hope that it will be useful, but WITHOUT
 606 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 607 # FOR A PARTICULAR PURPOSE.
 608 # See the GNU General Public License for more details.
 609 #
 610 # You should have received a copy of the GNU General Public License along with
 611 # this program.
 612 # If not, see L<http://www.gnu.org/licenses/>.
 613
 614 # vim: set ts=8 sw=8 sts=8 noet fo+=w tw=79 fenc=UTF-8 :