git-update-dirs

   1 #!/usr/bin/env perl
   2
   3 #==============================================================================
   4 # git-update-dirs
   5 # File ID: f1ba77e4-444e-11e0-963c-00023faf1383
   6 #
   7 # Update many git local repositories at once
   8 #
   9 # Character set: UTF-8
  10 # ©opyleft 2011– Øyvind A. Holm <sunny@sunbase.org>
  11 # License: GNU General Public License version 2 or later, see end of file for
  12 # legal stuff.
  13 #==============================================================================
  14
  15 use strict;
  16 use warnings;
  17 use Cwd qw{ abs_path getcwd };
  18 use Getopt::Long;
  19 use File::Basename;
  20
  21 local $| = 1;
  22
  23 our %Opt = (
  24
  25         'aggressive-compress' => 0,
  26         'all-options' => 0,
  27         'allbr' => 0,
  28         'compress' => 0,
  29         'dangling' => 0,
  30         'delete-dangling' => 0,
  31         'dry-run' => 0,
  32         'fetch' => 0,
  33         'fetch-prune' => 0,
  34         'ga-dropget' => 0,
  35         'ga-dropunused' => 0,
  36         'ga-getnew' => 0,
  37         'ga-moveunused' => 0,
  38         'ga-sync' => 0,
  39         'ga-update-desc' => 0,
  40         'help' => 0,
  41         'lpar' => 0,
  42         'pull' => 0,
  43         'push' => 0,
  44         'quiet' => 0,
  45         'recursive' => 0,
  46         'submodule' => 0,
  47         'test' => 0,
  48         'verbose' => 0,
  49         'version' => 0,
  50
  51 );
  52 our @opt_dirs_from = ();
  53 our @opt_exec_after = ();
  54 our @opt_exec_before = ();
  55
  56 our $progname = $0;
  57 $progname =~ s/^.*\/(.*?)$/$1/;
  58 our $VERSION = '0.6.5';
  59
  60 my $archive_disk = "seagate-3tb"; # FIXME: Hardcoding
  61
  62 Getopt::Long::Configure('bundling');
  63 GetOptions(
  64
  65         'aggressive-compress|C' => \$Opt{'aggressive-compress'},
  66         'all-options|A' => \$Opt{'all-options'},
  67         'allbr|a+' => \$Opt{'allbr'},
  68         'compress|c' => \$Opt{'compress'},
  69         'dangling|d' => \$Opt{'dangling'},
  70         'delete-dangling|D' => \$Opt{'delete-dangling'},
  71         'dirs-from=s' => \@opt_dirs_from,
  72         'dry-run|n' => \$Opt{'dry-run'},
  73         'exec-after|e=s' => \@opt_exec_after,
  74         'exec-before|E=s' => \@opt_exec_before,
  75         'fetch-prune|F' => \$Opt{'fetch-prune'},
  76         'fetch|f' => \$Opt{'fetch'},
  77         'ga-dropget|G' => \$Opt{'ga-dropget'},
  78         'ga-dropunused|u' => \$Opt{'ga-dropunused'},
  79         'ga-getnew|N' => \$Opt{'ga-getnew'},
  80         'ga-moveunused|U' => \$Opt{'ga-moveunused'},
  81         'ga-sync|g' => \$Opt{'ga-sync'},
  82         'ga-update-desc|S' => \$Opt{'ga-update-desc'},
  83         'help|h' => \$Opt{'help'},
  84         'lpar|l' => \$Opt{'lpar'},
  85         'pull|p' => \$Opt{'pull'},
  86         'push|P' => \$Opt{'push'},
  87         'quiet|q+' => \$Opt{'quiet'},
  88         'recursive|r' => \$Opt{'recursive'},
  89         'submodule|s' => \$Opt{'submodule'},
  90         'test|t' => \$Opt{'test'},
  91         'verbose|v+' => \$Opt{'verbose'},
  92         'version' => \$Opt{'version'},
  93
  94 ) || die("$progname: Option error. Use -h for help.\n");
  95
  96 $Opt{'verbose'} -= $Opt{'quiet'};
  97 $Opt{'help'} && usage(0);
  98 if ($Opt{'version'}) {
  99         print_version();
 100         exit(0);
 101 }
 102
 103 exit(main());
 104
 105 sub main {
 106         # {{{
 107         my $Retval = 0;
 108
 109         ($Opt{'ga-dropunused'} && $Opt{'ga-moveunused'}) &&
 110                 die("$progname: Can't use -u/--ga-dropunused and " .
 111                     "-U/--ga-moveunused at the same time\n");
 112
 113         if ($Opt{'all-options'}) {
 114                 $Opt{'allbr'} += 1;
 115                 $Opt{'dangling'} = 1;
 116                 $Opt{'fetch-prune'} = 1;
 117                 $Opt{'ga-sync'} = 1;
 118                 $Opt{'ga-update-desc'} = 1;
 119                 $Opt{'lpar'} = 1;
 120                 $Opt{'pull'} = 1;
 121                 $Opt{'push'} = 1;
 122                 $Opt{'submodule'} = 1;
 123         }
 124         ($Opt{'ga-dropget'} || $Opt{'ga-dropunused'}
 125             || $Opt{'ga-moveunused'}) &&
 126                 ($Opt{'ga-sync'} = 1);
 127
 128         my @Dirs = @ARGV;
 129         my @err_compress = ();
 130         my @err_dangling = ();
 131         my @err_fetch = ();
 132         my @err_gasync = ();
 133         my @err_pull = ();
 134         my @err_push = ();
 135         my @err_test = ();
 136         my @err_updsub = ();
 137
 138         if ($Opt{'recursive'}) {
 139                 my $repos = `find . -type d -name .git -print0`;
 140                 $repos =~ s/\/\.git\000/\000/g;
 141                 my %dupdir;
 142                 @Dirs = sort(grep { !$dupdir{$_}++ }
 143                         (@Dirs, split("\000", $repos)));
 144         }
 145
 146         my $orig_dir = getcwd();
 147         my ($total_before, $total_after, $total_saved) = (0, 0, 0);
 148         my ($totnum_before, $totnum_after) = (0, 0);
 149
 150         for my $dirfile (@opt_dirs_from) {
 151                 if ($dirfile eq '-') {
 152                         while (<STDIN>) {
 153                                 chomp();
 154                                 push(@Dirs, $_);
 155                         }
 156                 } else {
 157                         open(DirFP, "<$dirfile") or
 158                                 die("$progname: $dirfile: Cannot read " .
 159                                     "from file: $!\n");
 160                         while (<DirFP>) {
 161                                 chomp();
 162                                 push(@Dirs, $_);
 163                         }
 164                         close(DirFP);
 165                 }
 166         }
 167
 168         LOOP: for my $f (@Dirs) {
 169                 my $object_dir = '';
 170                 -d "$f/.git/." && ($object_dir = ".git/objects");
 171                 -d "$f/objects/." && ($object_dir = "objects");
 172                 msg(2, "object_dir = '$object_dir'");
 173                 if (length($object_dir)) {
 174                         $Opt{'verbose'} >= -1 &&
 175                                 print("================ $f " .
 176                                       "================\n");
 177                         if (!chdir($f)) {
 178                                 warn("$progname: $f: Cannot chdir: $!\n");
 179                                 next LOOP;
 180                         }
 181                         inside_git_dir() &&
 182                                 (chdir('..'), $object_dir = ".git/objects");
 183                         my $is_bare = (get_config('core.bare') =~ /^true/)
 184                                           ? 1
 185                                           : 0;
 186                         msg(2, "is_bare = '$is_bare'");
 187
 188                         my $Lh = "[0-9a-fA-F]";
 189                         my $uuid_templ = "$Lh\{8}-$Lh\{4}-$Lh\{4}-" .
 190                                          "$Lh\{4}-$Lh\{12}";
 191                         my $is_annex = (get_config('annex.uuid') =~
 192                                        /^$uuid_templ/)
 193                                            ? 1
 194                                            : 0;
 195                         msg(2, "is_annex = '$is_annex'");
 196
 197                         if (should('exec-before')) {
 198                                 for my $arg (@opt_exec_before) {
 199                                         mysystem($arg);
 200                                 }
 201                         }
 202                         if (should('lpar')) {
 203                                 mysystem("lpar");
 204                         }
 205                         if (should('test')) {
 206                                 mysystem("git", "fsck") && (
 207                                         push(@err_test, $f),
 208                                         warn("$progname: $f: ERRORS FOUND! " .
 209                                              "Skipping other actions " .
 210                                              "for this repo.\n"),
 211                                         next LOOP
 212                                 );
 213                         }
 214                         if (should('fetch-prune')) {
 215                                 mysystem("git", "fetch", "--all", "--prune") &&
 216                                         push(@err_fetch, $f);
 217                         } elsif (should('fetch')) {
 218                                 mysystem("git", "fetch", "--all") &&
 219                                         push(@err_fetch, $f);
 220                         }
 221                         if (should('pull') && !$is_bare) {
 222                                 mysystem("git", "pull", "--ff-only") &&
 223                                         push(@err_pull, $f);
 224                                 if (-e '.emptydirs') {
 225                                         mysystem('git', 'restore-dirs');
 226                                 }
 227                         }
 228                         if ($is_annex &&
 229                             (should('ga-sync') || should('ga-getnew'))) {
 230                                 should('ga-sync') && mysystem("ga", "sync") &&
 231                                         push(@err_gasync, $f);
 232                                 if (should('ga-dropget')) {
 233                                         mysystem("ga", "drop", "--auto");
 234                                         mysystem("ga", "sync");
 235                                 }
 236                                 if (should('ga-dropunused')) {
 237                                         mysystem("ga", "unused");
 238                                         mysystem("ga", "dropunused", "all");
 239                                         mysystem("ga", "sync");
 240                                 }
 241                                 if (should('ga-moveunused')) {
 242                                         if (open(my $fh, "git remote |")) {
 243                                                 my $found = 0;
 244                                                 while (my $curr_remote = <$fh>) {
 245                                                         if ($curr_remote =~ /^$archive_disk$/) {
 246                                                                 mysystem("ga", "unused");
 247                                                                 mysystem("ga", "move", "--unused",
 248                                                                          "--to", $archive_disk);
 249                                                                 mysystem("ga", "sync");
 250                                                                 last;
 251                                                         }
 252                                                 }
 253                                                 close($fh);
 254                                         } else {
 255                                                 warn("$progname: Cannot open " .
 256                                                      "'git remote' pipe: $!\n");
 257                                         }
 258                                 }
 259                                 if (should('ga-dropget') || should('ga-getnew')) {
 260                                         if (!$Opt{'ga-getnew'}) {
 261                                                 mysystem("ga", "get",
 262                                                          "--auto");
 263                                                 mysystem("ga", "sync");
 264                                         } else {
 265                                                 mysystem("ga-getnew");
 266                                         }
 267                                 }
 268                         }
 269                         if ($is_annex && should('ga-update-desc')) {
 270                                 mysystem("ga", "update-desc");
 271                         }
 272                         if (should('dangling')) {
 273                                 mysystem("git", "dangling") &&
 274                                         push(@err_dangling, $f);
 275                         }
 276                         if (should('allbr') &&
 277                             ($is_bare || $Opt{'allbr'} > 1)) {
 278                                 mysystem("git", "nobr");
 279                                 mysystem("git", "allbr", "-a");
 280                                 mysystem("git", "checkout", "-");
 281                         }
 282                         if (should('push')) {
 283                                 mysystem("git", "pa") && push(@err_push, $f);
 284                         }
 285                         if (should('submodule') && -e ".gitmodules") {
 286                                 mysystem("git", "submodule", "init");
 287                                 mysystem("git", "submodule", "update") &&
 288                                         push(@err_updsub, $f);
 289                         }
 290                         if (should('compress') ||
 291                             should('aggressive-compress')) {
 292                                 chomp(my $before =
 293                                     `(find $object_dir -type f -printf '%s+' ; echo 0) | bc`);
 294                                 $total_before += $before;
 295                                 chomp(my $numfiles_before =
 296                                     `find $object_dir -type f | wc -l`);
 297                                 $totnum_before += $numfiles_before;
 298                                 $Opt{'dry-run'} || print("\n");
 299                                 mysystem("git", "count-objects", "-vH");
 300                                 $Opt{'dry-run'} || print("\n");
 301                                 if (should('aggressive-compress')) {
 302                                         mysystem("git gc --aggressive") &&
 303                                                 push(@err_compress, $f);
 304                                 } else {
 305                                         mysystem("git gc") &&
 306                                                 push(@err_compress, $f);
 307                                 }
 308                                 chomp(my $after =
 309                                     `(find $object_dir -type f -printf '%s+' ; echo 0) | bc`);
 310                                 $total_after += $after;
 311                                 my $saved = $before - $after;
 312                                 chomp(my $numfiles_after =
 313                                       `find $object_dir -type f | wc -l`);
 314                                 $totnum_after += $numfiles_after;
 315                                 $before &&
 316                                         printf("\nBefore: %s\n" .
 317                                                "After : %s\n" .
 318                                                "Saved : %s (%.4f%%)\n",
 319                                                commify($before),
 320                                                commify($after),
 321                                                commify($saved),
 322                                                100.0 * $saved / $before);
 323                                 printf("Number of files in %s: " .
 324                                        "before: %u, after: %u, saved: %d\n",
 325                                        $object_dir,
 326                                        $numfiles_before,
 327                                        $numfiles_after,
 328                                        $numfiles_before-$numfiles_after)
 329                                 # Temporarily (?) disabled, it takes a heck of
 330                                 # a long time and uses loads of CPU and memory.
 331                                 # If the git-annex repo gets corrupted it's in
 332                                 # most cases good enough to delete
 333                                 # .git/annex/index anyway, and that can be done
 334                                 # manually.
 335                                 # $is_annex && should('dangling') && mysystem("ga", "repair");
 336                         }
 337                         if (should('delete-dangling') && !$is_bare) {
 338                                 mysystem("git", "dangling", "-D");
 339                         }
 340                         if (should('lpar')) {
 341                                 mysystem("lpar");
 342                         }
 343                         if (should('exec-after')) {
 344                                 for my $arg (@opt_exec_after) {
 345                                         mysystem($arg);
 346                                 }
 347                         }
 348                         $Opt{'verbose'} >= -1 && print("\n");
 349                 }
 350                 chdir($orig_dir) || die(
 351                     "$progname: $orig_dir: Cannot return to " .
 352                     "original directory: $!\n");
 353         }
 354         scalar(@err_fetch) && print("$progname: Unable to fetch from: " .
 355                                     join(" ", @err_fetch) . "\n");
 356         scalar(@err_gasync) && print("$progname: Unable to run ga sync: " .
 357                                      join(" ", @err_gasync) . "\n");
 358         scalar(@err_dangling) &&
 359                 print("$progname: Unable to run git dangling: " .
 360                       join(" ", @err_dangling) . "\n");
 361         scalar(@err_pull) && print("$progname: Unable to pull from: " .
 362                                    join(" ", @err_pull) . "\n");
 363         scalar(@err_push) && print("$progname: Unable to push from: " .
 364                                    join(" ", @err_push) . "\n");
 365         scalar(@err_updsub) &&
 366                 print("$progname: Unable to update submodules in: " .
 367                       join(" ", @err_updsub) . "\n");
 368         scalar(@err_compress) && print("$progname: Unable to compress: " .
 369                                        join(" ", @err_compress) . "\n");
 370         scalar(@err_test) && print("$progname: Error in git fsck: " .
 371                                    join(" ", @err_test) . "\n");
 372
 373         if ($Opt{'compress'} || $Opt{'aggressive-compress'}) {
 374                 my $total_saved = $total_before - $total_after;
 375                 printf("Before: %s\nAfter : %s\n",
 376                     commify($total_before), commify($total_after));
 377                 $total_before &&
 378                         printf("Total : %s (%.4f%%)\n",
 379                                commify($total_saved),
 380                                100.0 * $total_saved / $total_before);
 381                 printf("Number of object files: before: %u, after: %u, " .
 382                        "saved: %d\n", $totnum_before, $totnum_after,
 383                        $totnum_before-$totnum_after)
 384         }
 385
 386         return $Retval;
 387         # }}}
 388 }
 389
 390 sub check_sig {
 391         # {{{
 392         my $retval = shift;
 393         ($retval & 127) &&
 394                 die("\n$progname: Child process interrupted, aborting.\n");
 395         return(0);
 396         # }}}
 397 } # check_sig()
 398
 399 sub commify {
 400         # {{{
 401         my $Str = reverse $_[0];
 402         $Str =~ s/(\d\d\d)(?=\d)(?!\d*\,)/$1,/g;
 403         return scalar reverse $Str;
 404         # }}}
 405 } # commify()
 406
 407 sub get_config {
 408         # {{{
 409         my $name = shift;
 410         my $retval = '';
 411         chomp($retval = `git config --get "$name"`);
 412         return($retval);
 413         # }}}
 414 } # get_config()
 415
 416 sub inside_git_dir {
 417         # {{{
 418         my $basename = basename(abs_path(getcwd()));
 419         my $retval = ($basename eq '.git') ? 1 : 0;
 420         return($retval);
 421         # }}}
 422 } # inside_git_dir()
 423
 424 sub mysystem {
 425         # {{{
 426         my @cmd = @_;
 427         msg(0, sprintf("%s '", $Opt{'dry-run'} ? "Simulating" : "Executing") .
 428                                join(" ", @cmd) . "'...");
 429         $? = 0;
 430         !$Opt{'dry-run'} && system(@cmd) && check_sig($?);
 431         return $?;
 432         # }}}
 433 } # mysystem()
 434
 435 sub print_version {
 436         # Print program version {{{
 437         print("$progname $VERSION\n");
 438         return;
 439         # }}}
 440 }
 441
 442 sub should {
 443         # {{{
 444         my $name = shift;
 445         get_config("git-update-dirs.no-$name") eq 'true' && return 0;
 446         my $retval = 0;
 447         if ($name =~ /^exec-(before|after)$/) {
 448                 scalar($1 eq "before" ? @opt_exec_before : @opt_exec_after) &&
 449                         ($retval = 1);
 450         } else {
 451                 $Opt{$name} && ($retval = 1);
 452         }
 453         return($retval);
 454         # }}}
 455 } # should()
 456
 457 sub usage {
 458         # Send the help message to stdout {{{
 459         my $Retval = shift;
 460
 461         if ($Opt{'verbose'}) {
 462                 print("\n");
 463                 print_version();
 464         }
 465         print(<<"END");
 466
 467 Execute a predefined or customised set of commands in multiple local Git
 468 repositories in one operation.
 469
 470 Usage: $progname [options] [directories [...]]
 471
 472 Options, listed in the order they are executed in every Git repository:
 473
 474   -E X, --exec-before X
 475     Execute command X in every repo before all other commands. This
 476     option can be specified multiple times to run several commands.
 477     To disable: git config git-update-dirs.no-exec-before true
 478   -l, --lpar
 479     Execute lpar before and after fetch/pull and push
 480     To disable: git config git-update-dirs.no-lpar true
 481   -t, --test
 482     Test integrity of local repositories by running "git fsck".
 483     To disable: git config git-update-dirs.no-test true
 484   -F, --fetch-prune
 485     Fetch new commits from all remotes and prune deleted remote
 486     branches.
 487     To disable: git config git-update-dirs.no-fetch-prune true
 488   -f, --fetch
 489     Fetch new commits from all remotes.
 490     To disable: git config git-update-dirs.no-fetch true
 491   -p, --pull
 492     Also execute "git pull --ff-only".
 493     To disable: git config git-update-dirs.no-pull true
 494   -g, --ga-sync
 495     If the repo is used by git-annex, run "ga sync".
 496     To disable: git config git-update-dirs.no-ga-sync true
 497   -G, --ga-dropget
 498     Drop annex files having more copies than necessary, get files with
 499     fewer copies than necessary.
 500     To disable: git config git-update-dirs.no-ga-dropget true
 501   -u, --ga-dropunused
 502     In a git-annex repo, run "ga unused" followed by "ga dropunused
 503     all". Can't be used together with -U/--ga-moveunused.
 504     To disable: git config git-update-dirs.no-ga-dropunused true
 505   -U, --ga-moveunused
 506     Move unused git-annex contents to the '$archive_disk' remote. Can't
 507     be used together with -u/--ga-dropunused.
 508     To disable: git config git-update-dirs.no-ga-moveunused true
 509   -N, --ga-getnew
 510     Execute "ga-getnew", i.e. use "ga get --auto" to get all files from
 511     one month back that don't have enough copies.
 512     To disable: git config git-update-dirs.no-ga-getnew true
 513   -S, --ga-update-desc
 514     If the directory is controlled by git-annex, execute "ga
 515     update-desc" to set the description to the output from ga-pwd(1).
 516     To disable: git config git-update-dirs.no-ga-update-desc true
 517   -d, --dangling
 518     Execute "git dangling", i.e. turn all dangling commits into
 519     branches.
 520     To disable: git config git-update-dirs.no-dangling true
 521   -a, --allbr
 522     Execute "git nobr", "git allbr -a" and "git checkout -". If this
 523     option is specified once, it's only executed in bare repos. To also
 524     execute it in non-bare repos, it must be specified twice.
 525     To disable: git config git-update-dirs.no-allbr true
 526   -P, --push
 527     Also execute "git pa".
 528     To disable: git config git-update-dirs.no-push true
 529   -s, --submodule
 530     Update submodules if .gitmodules is found.
 531     To disable: git config git-update-dirs.no-submodule true
 532   -c, --compress
 533     Compress local repositories to save space.
 534     To disable: git config git-update-dirs.no-compress true
 535   -C, --aggressive-compress
 536     Use --aggressive when compressing the repository.
 537     To disable: git config git-update-dirs.no-aggressive-compress true
 538   -D, --delete-dangling
 539     Execute "git dangling -D" after execution to remove local commit-*
 540     branches and tag-* tags. This option is ignored in bare repos.
 541     To disable: git config git-update-dirs.no-delete-dangling true
 542   -e X, --exec-after X
 543     Execute command X in every repo after all other commands. This
 544     option can be specified multiple times to run several commands.
 545     To disable: git config git-update-dirs.no-exec-after true
 546
 547   -A, --all-options
 548     Run the program as if "-lFpgSdaPs" had been specified.
 549   --dirs-from X
 550     Read directory list from file X. If "-" is specified, read from
 551     stdin. Can be specified multiple times to read from different files.
 552   -h, --help
 553     Show this help.
 554   -n, --dry-run
 555     Simulate, don't actually execute any git commands.
 556   -q, --quiet
 557     Be more quiet. Can be repeated to increase silence.
 558   -r, --recursive
 559     Update all repositories recursively under the current directory.
 560   -v, --verbose
 561     Increase level of verbosity. Can be repeated.
 562   --version
 563     Print version information.
 564
 565 To disable some of these commands in a specific repository, set the git
 566 config variable git-update-dirs.no-OPTION to "true". For example, to
 567 disable push:
 568
 569   git config git-update-dirs.no-push true
 570
 571 Or disable aggressive compression:
 572
 573   git config git-update-dirs.no-aggressive-compress true
 574
 575 In this case aggressive compression will be disabled, and it will fall
 576 back to regular compression.
 577
 578 Only the value "true" (with lower case letters) is recognised, any other
 579 value will allow the command to run.
 580
 581 END
 582         exit($Retval);
 583         # }}}
 584 }
 585
 586 sub msg {
 587         # Print a status message to stderr based on verbosity level {{{
 588         my ($verbose_level, $Txt) = @_;
 589
 590         if ($Opt{'verbose'} >= $verbose_level) {
 591                 print(STDERR "$progname: $Txt\n");
 592         }
 593         return;
 594         # }}}
 595 }
 596
 597 __END__
 598
 599 # This program is free software; you can redistribute it and/or modify it under
 600 # the terms of the GNU General Public License as published by the Free Software
 601 # Foundation; either version 2 of the License, or (at your option) any later
 602 # version.
 603 #
 604 # This program is distributed in the hope that it will be useful, but WITHOUT
 605 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 606 # FOR A PARTICULAR PURPOSE.
 607 # See the GNU General Public License for more details.
 608 #
 609 # You should have received a copy of the GNU General Public License along with
 610 # this program.
 611 # If not, see L<http://www.gnu.org/licenses/>.
 612
 613 # vim: set ts=8 sw=8 sts=8 noet fo+=w tw=79 fenc=UTF-8 :