bin/lo-pack-sources

   1 #!/usr/bin/perl
   2     eval 'exec /usr/bin/perl -S $0 ${1+"$@"}'
   3         if $running_under_some_shell;
   4 #!/usr/bin/perl
   5
   6 use strict;
   7 use File::Copy;
   8 use File::Temp qw/ tempfile tempdir /;
   9
  10 my %module_dirname = (
  11     "core"              => "",
  12     "dictionaries"      => "dictionaries",
  13     "help"              => "helpcontent2",
  14     "translations"      => "translations"
  15 );
  16 my $lo_topdir_name;
  17
  18 # get libreoffice-build version from the given libreoffice-build sources
  19 sub get_config_version($)
  20 {
  21     my ($lo_core_dir) = @_;
  22     my $version;
  23
  24     open (CONFIGURE, "$lo_core_dir/configure.ac") ||
  25         die "can't open \"$lo_core_dir/configure.ac\" for reading: $!\n";
  26
  27     while (my $line = <CONFIGURE>) {
  28         chomp $line;
  29
  30         if ($line =~ /AC_INIT\s*\(\s*libreoffice-build\s*,\s*([\w\.]*)\)/) {
  31             $version="$1";
  32         }
  33     }
  34     close (CONFIGURE);
  35     return $version;
  36 }
  37
  38 # increment the version for a test build:
  39 #        + add 'a' if the version ended with a number
  40 #       + bump the letter otherwise
  41 sub inc_test_version($)
  42 {
  43     my ($version) = @_;
  44
  45     my $lastchar = chop $version;
  46     my $new_version;
  47
  48     if ($lastchar =~ /\d/) {
  49         return "$version" . "$lastchar" . "a";
  50     } elsif ($lastchar =~ /\w/) {
  51         # select next letter alphabetically: a->b, b->c, ...
  52         $lastchar =~ tr/0a-zA-Z/a-zA-Z0/;
  53         return "$version" . "$lastchar";
  54     } else {
  55         die "Can't generate test version from \"$version$lastchar\n";
  56     }
  57 }
  58
  59 sub get_release_version($$$$)
  60 {
  61     my ($config_version, $state_config_version, $state_release_version, $inc_version) = @_;
  62     my $release_version;
  63
  64     if (defined $state_config_version &&
  65         defined $state_release_version &&
  66         "$state_config_version" eq "$config_version") {
  67         $release_version = "$state_release_version";
  68     } else {
  69         $release_version = "$config_version";
  70     }
  71
  72     if ( defined $inc_version ) {
  73         $release_version = inc_test_version($release_version);
  74     }
  75
  76     return $release_version;
  77 }
  78
  79 # copy files to temp dir; showing a progress; using a black list
  80 sub copy_dir_filter_and_show_progress($$)
  81 {
  82     my ($source_dir, $target_dir) = @_;
  83
  84     print "Copying \"$source_dir\" -> \"$target_dir\"...";
  85     # copy sources from git and show progress
  86     system ("cd $source_dir && " .
  87             "git archive --format=tar HEAD | " .
  88             "  tar -xf - -C $target_dir ") &&
  89         die "Error: copying failed: $!\n";
  90     print "\n";
  91 }
  92
  93 # copy files to temp dir; showing a progress; using a black list
  94 sub remove_empty_submodules($)
  95 {
  96     my ($target_topdir) = @_;
  97
  98     foreach my $submodule (sort values %module_dirname) {
  99         next unless ($submodule);
 100         print "Removing empty submodule: $submodule...\n";
 101         rmdir "$target_topdir/$submodule" || die "Error: Can't remove submodule directory: $target_topdir/$submodule";
 102     }
 103 }
 104
 105 # copy the source directory into a tmp directory
 106 # omit the .git subdirectories
 107 sub copy_lo_module_to_tempdir($$$)
 108 {
 109     my ($source_dir, $module, $lo_topdir_name) = @_;
 110     my $tempdir = tempdir( 'libreoffice-XXXXXX', DIR => File::Spec->tmpdir );
 111
 112     mkdir "$tempdir/$lo_topdir_name" || die "Can't create directory \"$tempdir/$lo_topdir_name\": $!\n";
 113     mkdir "$tempdir/$lo_topdir_name/$module_dirname{$module}" || die "Can't create directory \"$tempdir/$lo_topdir_name/$module_dirname{$module}\": $!\n" if ($module_dirname{$module});
 114
 115     copy_dir_filter_and_show_progress("$source_dir/$module_dirname{$module}", "$tempdir/$lo_topdir_name/$module_dirname{$module}");
 116     remove_empty_submodules("$tempdir/$lo_topdir_name/") if ($module eq "core");
 117
 118     return $tempdir;
 119 }
 120
 121 sub generate_lo_module_changelog($$$)
 122 {
 123     my ($source_dir, $lo_module_release_topdir, $module) = @_;
 124
 125     my $log_name = "ChangeLog";
 126     $log_name .= "-$module_dirname{$module}" if ($module_dirname{$module});
 127     print "Generating changelog for $module...\n";
 128     system ("cd $source_dir/$module_dirname{$module} && " .
 129             "git log --date=short --pretty='format:%cd  %an  <%ae>  [%H]%n%n%w(0,8,8)%s%n%e%+b' " .
 130             ">$lo_module_release_topdir/$log_name" ) &&
 131         die "Error: generating failed: $!\n";
 132 }
 133
 134 sub run_autogen($$)
 135 {
 136     my ($dir, $module) = @_;
 137
 138     print "Running autogen for $module...\n";
 139     system ("cd $dir && " .
 140             "NOCONFIGURE=1 ./autogen.sh && " .
 141             "rm -rf autom4te.cache && " .
 142             "cd - >/dev/null 2>&1") && die "Error: autogen failed: $!\n";
 143 }
 144
 145 sub generate_sources_version_file($$)
 146 {
 147     my ($dir, $release_version) = @_;
 148
 149     open (VERFILE, ">$dir/sources.ver") || die "Can't open $dir/sources.ver: $!\n";
 150
 151     print VERFILE "lo_sources_ver=$release_version\n";
 152
 153     close VERFILE;
 154 }
 155
 156 sub generate_tarball($$$)
 157 {
 158     my ($dir, $tarball, $compressor_option) = @_;
 159
 160     print "Creating $tarball...";
 161     # generate the tarball in the current directory; avoid "./" prefix in the stored paths; show progress
 162     system ("tar -c -C $dir $lo_topdir_name | $compressor_option $tarball") &&
 163         die "Error: releasing failed: $!\n";
 164     print "\n";
 165 }
 166
 167 sub generate_md5($)
 168 {
 169     my ($filename) = @_;
 170
 171     print "Generating MD5...\n";
 172     system ("md5sum $filename >$filename.md5") &&
 173         die "Error: releasing failed: $!\n";
 174 }
 175
 176 sub default_releases_state_file($)
 177 {
 178     my ($lo_core_dir) = @_;
 179
 180     my $rootdir = $lo_core_dir;
 181     $rootdir =~ s/^(.*?)\/?[^\/]+\/?$/$1/;
 182
 183     my $releases_state_file;
 184     if ($rootdir) {
 185         $releases_state_file = "$rootdir/.releases";
 186     } else {
 187         $releases_state_file = ".releases";
 188     }
 189
 190     return "$releases_state_file";
 191 }
 192
 193 sub load_releases_state($)
 194 {
 195     my ($releases_state_file) = @_;
 196
 197     my $state_config_version;
 198     my $state_release_version;
 199
 200     if (open (STATE, "$releases_state_file")) {
 201
 202         while (my $line = <STATE>) {
 203             chomp $line;
 204
 205             if ($line =~ /^\s*configure_version\s*=\s*(.*)$/) {
 206                 $state_config_version = "$1";
 207             } elsif ($line =~ /^\s*released_version\s*=\s*(.*)$/) {
 208                 $state_release_version = "$1";
 209             }
 210         }
 211             close (STATE);
 212     }
 213
 214     return $state_config_version, $state_release_version;
 215 }
 216
 217 sub save_releases_state($$$)
 218 {
 219     my ($releases_state_file, $config_version, $release_version) = @_;
 220
 221     open (STATE, '>', "$releases_state_file") ||
 222         die "Can't open \"$releases_state_file\" for writing: $!\n";
 223
 224     print STATE "configure_version = $config_version\n";
 225     print STATE "released_version  = $release_version\n";
 226
 227     close (STATE);
 228 }
 229
 230 sub remove_tempdir($)
 231 {
 232     my ($tempdir) = @_;
 233
 234 #    print "Cleaning $tempdir...\n";
 235     system ("rm -rf $tempdir") && die "Error: rm failed: $!\n";
 236 }
 237
 238 sub check_if_file_exists($$)
 239 {
 240     my ($file, $force) = @_;
 241
 242     if (-e $file) {
 243         if (defined $force) {
 244             print "Warning: $file already exists and will be replaced!\n";
 245         } else {
 246             die "Error: $file already exists.\n".
 247                 "       Use --force if you want to replace it.\n";
 248         }
 249     }
 250 }
 251
 252 sub check_if_already_released($$$$$$)
 253 {
 254     my ($p_module_tarball_name, $force, $bzip2, $xz, $pack_lo_core, $pack_lo_modules) = @_;
 255
 256     foreach my $tarball_name ( sort values %{$p_module_tarball_name} ) {
 257         check_if_file_exists("$tarball_name.tar.bz2", $force) if (defined $bzip2);
 258         check_if_file_exists("$tarball_name.tar.xz", $force) if (defined $xz);
 259     }
 260 }
 261
 262 sub prepare_module_sources($$$$)
 263 {
 264     my ($source_dir, $release_version, $module, $lo_topdir_name) = @_;
 265
 266     # prepare sources
 267     my $temp_dir = copy_lo_module_to_tempdir($source_dir, $module, $lo_topdir_name);
 268     generate_lo_module_changelog($source_dir, "$temp_dir/$lo_topdir_name", $module);
 269     run_autogen("$temp_dir/$lo_topdir_name", $module) if ($module eq 'core');
 270     generate_sources_version_file("$temp_dir/$lo_topdir_name", $release_version) if ($module eq 'core');
 271
 272     return $temp_dir;
 273 }
 274
 275 sub pack_module_sources($$$$)
 276 {
 277     my ($temp_dir, $md5, $tarball, $compressor_option) = @_;
 278
 279     generate_tarball($temp_dir, $tarball, $compressor_option);
 280     generate_md5($tarball) if (defined $md5);
 281 }
 282
 283 sub generate_module_tarball($$$$$$$$)
 284 {
 285     my ($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $module_tarball_name) = @_;
 286          my $PARALLELISM =  $ENV{'PARALLELISM'};
 287          my $CPUS_XZ = 0;
 288          my $CPUS_BZ2 = "";
 289          my $bzip_arguments;
 290
 291          # Set CPUS_ to the number of CPUs that should be used. If PARALLELISM
 292     # is set then this is used otherwise autodetect is used.
 293     if (defined $PARALLELISM) {
 294         if ($PARALLELISM > 0) {
 295             $CPUS_XZ = $PARALLELISM;
 296             $CPUS_BZ2 = "-p$PARALLELISM";
 297         } else {
 298             $CPUS_XZ = 1;
 299             $CPUS_BZ2 = "-p1";
 300                   }
 301     }
 302
 303          if (defined $bzip2) {
 304                   my $exit_code = system("pbzip2 --version >/dev/null 2>&1");
 305                   if ($exit_code == 0) {
 306                           $bzip_arguments = "pbzip2 -z -b20 $CPUS_BZ2";
 307                   } else {
 308                           $bzip_arguments = "bzip2 -z --best";
 309                           print("Consider installing pbzip2, using bzip2 now.\n");
 310                   }
 311          }
 312
 313     my $temp_dir = prepare_module_sources($source_dir, $release_version, $module, $lo_topdir_name);
 314     pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "$bzip_arguments > ") if (defined $bzip2);
 315     pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z -T$CPUS_XZ -e > ") if (defined $xz);
 316     remove_tempdir($temp_dir);
 317 }
 318
 319
 320 sub generate_tarballs($$$$$$$$$)
 321 {
 322     my ($source_dir, $release_version, $md5, $bzip2, $xz, $lo_topdir_name, $p_module_tarball_name, $pack_lo_core, $pack_lo_modules) = @_;
 323
 324     foreach my $module (sort keys %{$p_module_tarball_name} ) {
 325         print "\n--- Generating $module ---\n";
 326         generate_module_tarball($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $p_module_tarball_name->{$module});
 327     }
 328 }
 329
 330
 331 sub usage()
 332 {
 333     print "This tool helps to pack the libreoffice-build and module sources\n\n" .
 334
 335           "Usage:\n".
 336           "\tlo-pack-sources [--help]\n" .
 337           "\t  [--force] [--md5] [--bzip2] [--xz]\n" .
 338           "\t [--version][--set-version=<ver>] [--inc-version]\n" .
 339           "\t [--no-submodule] [--module=<module>]\n" .
 340           "\t [dir]\n\n" .
 341
 342           "Options:\n\n" .
 343           "\t--help: print this help\n" .
 344           "\t--force: replace an already existing release of the same version\n" .
 345           "\t--md5: generate md5 sum for the final tarball\n" .
 346           "\t--bzip2: generate tarballs compressed by bzip2\n" .
 347           "\t--xz: generate tarballs compressed by xz (default)\n" .
 348           "\t--version: just print version of the released package but do not\n" .
 349           "\t\trelease it; the version is affected by the other options, e.g.\n" .
 350           "\t\t--inc-version\n" .
 351           "\t--set-version: force another version\n" .
 352           "\t--inc-version: increment the latest version; there is a difference\n" .
 353           "\t\tbetween test release (default) and final (not yet supported)\n" .
 354           "\t--no-submodule: do not pack sources from git submodules\n" .
 355           "\t--module=<module>: pack just a single module, use \"core\"\n" .
 356           "\t\tfor the main git repo,\n" .
 357           "\tdir: path of the source directory, either libreoffice-build or module\n";
 358 }
 359
 360
 361 my $module;
 362 my $ptf;
 363 my $md5;
 364 my $bzip2;
 365 my $xz;
 366 my $inc_version;
 367 my $config_version;
 368 my $set_version;
 369 my $get_config_version;
 370 my $release_version;
 371 my $pack_lo_core=1;
 372 my $pack_lo_modules=1;
 373 my $source_dir;
 374 my $releases_state_file;
 375 my $state_config_version;
 376 my $state_release_version;
 377 my $lo_core_tempdir;
 378 my $force;
 379 my $verbose=1;
 380 my %module_tarball_name;
 381
 382 ###################
 383 # Arguments parsing
 384 ###################
 385
 386 for my $arg (@ARGV) {
 387     if ($arg eq '--help' || $arg eq '-h') {
 388         usage;
 389         exit 0;
 390     } elsif ($arg eq '--force') {
 391         $force=1;
 392     } elsif ($arg eq '--md5') {
 393         $md5=1;
 394     } elsif ($arg eq '--bzip2') {
 395         $bzip2=1;
 396     } elsif ($arg eq '--xz') {
 397         $xz=1;
 398     } elsif ($arg eq '--version') {
 399         $get_config_version=1;
 400         $verbose = undef;
 401     } elsif ($arg eq '--inc-version') {
 402         $inc_version=1
 403     } elsif ($arg =~ m/--set-version=(.*)/) {
 404           $set_version="$1";
 405     } elsif ($arg eq '--no-submodule') {
 406         $module = "core";
 407     } elsif ($arg =~ m/--module=(.*)/) {
 408           # process just one module and do not pack libreoffice-build
 409           die("Error: unknown module: $1") unless (defined $module_dirname{$1});
 410           $module = $1;
 411     } elsif ($arg =~ /^-/ ) {
 412         die "Error: unknown option: $arg\n";
 413     } else {
 414         if (! defined $source_dir) {
 415             $source_dir = $arg;
 416         } else {
 417             die "Error: Too many arguments $arg\n";
 418         }
 419     }
 420 }
 421
 422 # ugly hack; we want only one module
 423 if ($module) {
 424     my $name = $module_dirname{$module};
 425     %module_dirname = ();
 426     $module_dirname{$module} = $name;
 427 }
 428
 429 ###################
 430 # Initial checks
 431 ###################
 432
 433 unless ( defined $source_dir ) {
 434     die "Error: undefined source directory, try --help\n";
 435 }
 436
 437 unless ( -d "$source_dir" ) {
 438     die "Error: is not a directory: $source_dir\n";
 439 }
 440
 441 # check if it is a valid libreoffice-core directory
 442 unless (-f "$source_dir/autogen.sh" && -f "$source_dir/config_host.mk.in") {
 443     die "Error: \"$source_dir\" is not a valid libreoffice-core directory\n";
 444 }
 445
 446 if (defined $set_version && defined $inc_version) {
 447     die "Error: --set-version and --inc-version options can't be used together\n";
 448 }
 449
 450 # default compression
 451 $xz = 1 unless (defined $xz || defined $bzip2);
 452
 453
 454 ###################
 455 # Main logic
 456 ###################
 457
 458
 459 print "Source: $source_dir\n" if ($verbose);
 460
 461 # detect some paths
 462 $releases_state_file = default_releases_state_file($source_dir) unless (defined $releases_state_file);
 463
 464 # detect versions
 465 $config_version = get_config_version($source_dir);
 466 ($state_config_version, $state_release_version) = load_releases_state($releases_state_file);
 467 if (defined $set_version) {
 468     $release_version = "$set_version";
 469 } else {
 470     $release_version = get_release_version($config_version, $state_config_version, $state_release_version, $inc_version);
 471 }
 472
 473 # define tarball names
 474 print "Detected module:\n";
 475 foreach my $module (sort keys %module_dirname) {
 476     if (-e "$source_dir/$module_dirname{$module}/.git") {
 477         print "    $module\n";
 478         if ($module eq "core") {
 479             $module_tarball_name{$module} = "libreoffice-$release_version";
 480         } else {
 481             $module_tarball_name{$module} = "libreoffice-$module-$release_version";
 482         }
 483     } else {
 484         print "did not found: $source_dir/$module_dirname{$module}/.git\n";
 485         print "Warning: $module sources are not available -> skipping\n";
 486     }
 487 }
 488
 489 # top directory inside the source tarballs
 490 $lo_topdir_name = "libreoffice-$release_version";
 491
 492 print "Default version   : $config_version\n" if ($verbose && defined $config_version);
 493 print "Last used version : $state_release_version\n" if ($verbose && defined $state_release_version);
 494 print "New version       : $release_version\n" if ($verbose);
 495
 496 # do the real job
 497 if ( defined $get_config_version ) {
 498     print "$release_version\n";
 499 } else {
 500     check_if_already_released(\%module_tarball_name, $force, $bzip2, $xz, $pack_lo_core, $pack_lo_modules);
 501
 502     # give a chance to stop the process
 503     print ("\nWaiting 3 seconds...\n");
 504     sleep 3;
 505
 506     generate_tarballs($source_dir, $release_version, $md5, $bzip2, $xz, $lo_topdir_name, \%module_tarball_name, $pack_lo_core, $pack_lo_modules);
 507
 508     if ( defined $releases_state_file ) {
 509         save_releases_state($releases_state_file, $config_version, $release_version);
 510     }
 511 }