lib/Fedora/Rebuild.pm

   1 package Fedora::Rebuild;
   2 use strict;
   3 use warnings;
   4 use version 0.77; our $VERSION = version->declare("v0.8.0");
   5
   6 use Moose;
   7 use Moose::Util::TypeConstraints;
   8 use Carp;
   9 use IO::Handle;
  10 use Fedora::Rebuild::Types qw( Mode );
  11 use Fedora::Rebuild::Set::Package;
  12 use Fedora::Rebuild::Package;
  13 use Fedora::Rebuild::Repository;
  14 use Fedora::Rebuild::Scheduler;
  15 use Fedora::Rebuild::Solver;
  16 use namespace::clean;
  17
  18 has 'all' => (is => 'ro', isa => 'Str', required => 1);
  19 has 'done' => (is => 'ro', isa => 'Str', required => 1);
  20 has 'failed' => (is => 'ro', isa => 'Str', required => 1);
  21 has 'workdir' => (is => 'ro', isa => 'Str', required => 1);
  22 # Directory where repository with built packages live. This is needed for
  23 # `mock' build mode.
  24 has 'repodir' => (is => 'ro', isa => 'Str', required => 1);
  25 # Git branch name
  26 # "f14", "f15" etc. Use "rawhide" for latest one.
  27 has 'dist' => (is => 'ro', isa => 'Str', required => 1);
  28 # Build target name
  29 # "dist-f14", "dist-f15" etc. Use "dist-rawhide" for latest one.
  30 has 'target' => (is => 'ro', isa => 'Str', required => 1);
  31 has 'message' => (is => 'ro', isa => 'Str', required => 1);
  32 # Reference to function with three arguments (BuildRequire name, relation
  33 # flag, version) returning false the BuildRequire should be considered, true
  34 # otherwise. If attribute is undef or missing, no filtering will be performed
  35 # (i.e. the same effect as sub {1}).
  36 has 'buildrequiresfilter' => (is => 'ro', isa => 'CodeRef', required => 0);
  37 # Pass "local" committing and building locally only, pass "koji" for pushing
  38 # commits and building in Koji. Default is "koji" to build in Koji.
  39 has 'mode' => (is => 'ro', isa => Mode, required => 0, default => 'koji');
  40 # Run rebuild in given number of threads. Default is 1.
  41 has 'threads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  42         message {"Attribute threads must be positive integer (was $_)"}),
  43     required => 0);
  44 # Load binary provides of already done packages or buildrequires in given
  45 # number of threads. This is to lower local I/O load. Default is 1.
  46 has 'loadthreads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  47         message {"Attribute loadthreads must be positive integer (was $_)"}),
  48     required => 0);
  49 # Select rebuildable packages in given number of threads.
  50 # Dependency solver is CPU intentsive work.
  51 # Default is 1.
  52 has 'selectthreads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  53         message {"Attribute selectthreads must be positive integer (was $_)"}),
  54     required => 0);
  55 # Maximal count of immediately sequential failures to accept and continue
  56 # rebuild process. If the limit exceeds, rebuild process will terminate. This
  57 # is good to catch pathologic cases when something is obviously wrong and
  58 # should be fixed before rebuilding (e.g. Koji is down, or you have not SSH
  59 # key loaded into SSH agent). Use non-positive number to disable this check.
  60 # Default value is 0 (i.e. not to check).
  61 has 'failurelimit' => (is => 'ro', isa => 'Int', required => 0, default => 0);
  62 # Build packages in dependency order. Default is true.
  63 # If set to false, the packages will be built irrespective to dependecies
  64 # (build- and run-time).
  65 has 'ordered' => (is => 'ro', isa => 'Bool', required => 0, default => 1);
  66
  67 has 'remaining_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  68     lazy_build => 1, init_arg => undef);
  69 has 'done_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  70     lazy_build => 1, init_arg => undef);
  71 has 'failed_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  72     lazy_build => 1, init_arg => undef);
  73 has 'subsequent_failures' => (is => 'rw', isa => 'Int', default => 0,
  74     init_arg => undef);
  75 has 'last_failed' => (is => 'rw', isa => 'Bool', default => 0,
  76     init_arg => undef);
  77 has 'repository' => (is => 'ro', isa => 'Fedora::Rebuild::Repository',
  78     lazy_build => 1, init_arg => undef);
  79
  80
  81 # Creates set of packages already rebuilt.
  82 sub _build_done_packages {
  83     my $self = shift;
  84     my $packages = Fedora::Rebuild::Set::Package->new();
  85     my $file;
  86
  87     if (not -e $self->done) {
  88         print "No packages have been rebuilt yet.\n";
  89         return $packages;
  90     }
  91
  92     open($file, '<', $self->done) or
  93         croak "Could not open `" . $self->done .
  94             "' for loading list of already rebuilt packages: $!";
  95     print "Loading list of already rebuilt package names...\n";
  96     while (local $_ = <$file>) {
  97         chomp;
  98         if (m/^\s*$/) { next; }
  99
 100         my $package = Fedora::Rebuild::Package->new(name => $_,
 101             workdir => $self->workdir, dist => $self->dist,
 102             target => $self->target, message => $self->message);
 103         $packages->insert($package);
 104         $self->repository->insert($package);
 105     }
 106     if ($!) {
 107         croak "Could not read list of rebuilt package names from file `" .
 108         $self->done . "': $!";
 109     }
 110     close $file;
 111     $self->repository->update;
 112     print "Number of done packages: " . $packages->size() . "\n";
 113
 114     return $packages;
 115 }
 116
 117
 118 sub mark_done {
 119     my ($self, $package) = @_;
 120     my $file = IO::Handle->new();
 121
 122     open($file, '>>', $self->done) or
 123         croak "Could not open `" . $self->done . "' for appending: $!";
 124     printf $file $package->name . "\n" or
 125         croak "Could not write data into `" . $self->done . "': $!";
 126     $file->flush && $file->sync && close($file) or
 127         croak "Could not flush and close `" . $self->done . "': $!";
 128
 129     $self->done_packages->insert($package);
 130     $self->last_failed(0);
 131     $self->subsequent_failures(0);
 132 }
 133
 134 # Creates set of packages not yet rebuilt not already failed.
 135 sub _build_remaining_packages {
 136     my $self = shift;
 137     my $packages = Fedora::Rebuild::Set::Package->new();
 138     my $file;
 139
 140     open($file, '<', $self->all) or
 141         croak "Could not open " . $self->all .
 142             " for loading list of package names to rebuild: $!";
 143     print "Loading list of all package names to rebuild...\n";
 144
 145     while (local $_ = <$file>) {
 146         chomp;
 147         if (m/^\s*$/) { next; }
 148         if ($packages->contains($_)) { next; }
 149         if (! $self->done_packages->contains($_) &&
 150                 ! $self->failed_packages->contains($_)) {
 151             $packages->insert(Fedora::Rebuild::Package->new(name => $_,
 152                 workdir => $self->workdir, dist => $self->dist,
 153                 target => $self->target, message => $self->message));
 154         }
 155     }
 156     if ($!) {
 157         croak "Could not read list of package names to rebuild from file `" .
 158             $self->all . "' :$!";
 159     }
 160     close $file;
 161
 162     print "Number of all packages: " .
 163         ($packages->size + $self->done_packages->size
 164             + $self->failed_packages->size) . "\n";
 165     print "Number of not yet rebuilt packages: " . $packages->size() . "\n";
 166     return $packages;
 167 }
 168
 169
 170 # Set of packages whose rebuild failed.
 171 sub _build_failed_packages {
 172     my $self = shift;
 173     my $packages = Fedora::Rebuild::Set::Package->new();
 174     my $file;
 175
 176     if (not -e $self->failed) {
 177         print "No packages have failed yet.\n";
 178         return $packages;
 179     }
 180
 181     open($file, '<', $self->failed) or
 182         croak "Could not open `" . $self->failed .
 183             "' for loading list of already failed packages: $!";
 184     print "Loading list of already failed package names...\n";
 185     while (local $_ = <$file>) {
 186         chomp;
 187         if (m/^\s*$/) { next; }
 188
 189         my $package = Fedora::Rebuild::Package->new(name => $_,
 190             workdir => $self->workdir, dist => $self->dist,
 191             target => $self->target, message => $self->message);
 192         $packages->insert($package);
 193     }
 194     if ($!) {
 195         croak "Could not read list of failed package names from file `" .
 196         $self->failed . "': $!";
 197     }
 198     close $file;
 199     print "Number of failed packages: " . $packages->size() . "\n";
 200
 201     return $packages;
 202 }
 203
 204 sub _build_repository {
 205     my $self = shift;
 206     return Fedora::Rebuild::Repository->new(path => $self->repodir);
 207 }
 208
 209 # Record package names into log of failed packages
 210 sub mark_failed {
 211     my ($self, $package) = @_;
 212     my $file = IO::Handle->new();
 213
 214     # Log failure
 215     open($file, '>>', $self->failed) or
 216         croak "Could not open `" . $self->failed . "' for appending: $!";
 217     printf $file $package->name . "\n" or
 218         croak "Could not write data into `" . $self->failed . "': $!";
 219     $file->flush && $file->sync && close($file) or
 220         croak "Could not flush and close `" . $self->failed . "': $!";
 221
 222     # Move to list of failed
 223     $self->failed_packages->insert($package);
 224
 225     # Check for very failures
 226     if ($self->last_failed) {
 227         $self->subsequent_failures($self->subsequent_failures + 1);
 228     } else {
 229         $self->last_failed(1);
 230         $self->subsequent_failures(1);
 231     }
 232     if ($self->failurelimit > 0 &&
 233             $self->subsequent_failures > $self->failurelimit) {
 234         croak "More then " . $self->failurelimit .
 235             " package(s) failed subsequently which is more then set " .
 236             "threshold. Aborting now.\n";
 237     }
 238 }
 239
 240 # Load build-requires for each not-yet done package.
 241 # Return true in case of success or croaks in case of failure.
 242 sub load_sourcedependencies {
 243     my $self = shift;
 244     my @packages = $self->remaining_packages->packages;
 245     my $scheduler = Fedora::Rebuild::Scheduler->new(
 246         limit => $self->loadthreads,
 247         name => 'Loading build-requires',
 248         total => $#packages
 249     );
 250     my %jobs = ();
 251     my $i = 0;
 252
 253     print "Loading build-time dependenices of not yet rebuilt packages...\n";
 254
 255     foreach my $package (@packages) {
 256         my $job = $scheduler->schedule($package->can('get_buildrequires'),
 257             $package, $self->mode);
 258         if (! defined $job) { next; }
 259         $jobs{$job} = $package;
 260         my %finished = $scheduler->finish(++$i < @packages);
 261
 262         while (my ($job, $status) = each %finished) {
 263             my $package = $jobs{$job};
 264             if (!$$status[0]) {
 265                 print "Could not load build-time dependencies for not yet " .
 266                     "built package `" . $package->name . "'.\n";
 267                 print "Waiting for finishing scheduled jobs...\n";
 268                 $scheduler->finish(1);
 269                 print "All jobs loading build-time dependcies have finished.\n";
 270                 croak "Could not load build-time dependencies.\n";
 271             }
 272         }
 273     }
 274
 275     print "Build-time dependencies of not-yet rebuilt packages loaded " .
 276         "successfully.\n";
 277     return 1;
 278 }
 279
 280
 281 # Load binary requires and provides of each done package.
 282 # Return true in case of success or croaks in case of failure.
 283 sub load_binarydependencies {
 284     my $self = shift;
 285     my @packages = $self->done_packages->packages;
 286     my $scheduler = Fedora::Rebuild::Scheduler->new(
 287         limit => $self->loadthreads,
 288         name => 'Loading binary dependendies',
 289         total => $#packages
 290     );
 291     my %jobs = ();
 292     my $i = 0;
 293
 294     print "Loading binary dependenices of already rebuilt packages...\n";
 295
 296     foreach my $package (@packages) {
 297         my $job = $scheduler->schedule($package->can('get_binarydependencies'),
 298             $package, $self->mode);
 299         if (! defined $job) { next; }
 300         $jobs{$job} = $package;
 301         my %finished = $scheduler->finish(++$i < @packages);
 302
 303         while (my ($job, $status) = each %finished) {
 304             my $package = $jobs{$job};
 305             if (!$$status[0]) {
 306                 print "Could not load binary dependencies for already built " .
 307                     "package `" . $package->name . "'.\n";
 308                 print "Waiting for finishing scheduled jobs...\n";
 309                 $scheduler->finish(1);
 310                 print "All jobs loading binary dependcies have finished.\n";
 311                 croak "Could not load binary dependencies\n";
 312             }
 313         }
 314     }
 315
 316     print "Binary dependencies of already rebuilt packages loaded " .
 317         "successfully.\n";
 318     return 1;
 319 }
 320
 321
 322 # Decides a package is rebuildable now.
 323 # Return 0 for false, 1 or true, undef for error while deciding.
 324 sub is_rebuildable {
 325     my ($self, $package) = @_;
 326     my $is_rebuildable;
 327     my $message = '';
 328
 329     if ($self->ordered) {
 330         my $solver = Fedora::Rebuild::Solver->new(
 331             'packages' => $self->done_packages,
 332             'dependencyfilter' => $self->buildrequiresfilter);
 333
 334         $is_rebuildable = $solver->is_buildable($package, \$message);
 335     } else {
 336         $is_rebuildable = 1;
 337         $message = "Package `" . $package->name .
 338             "' is buildable because unordered build mode is selected.";
 339     }
 340
 341     $package->log_is_rebuildable($is_rebuildable, $message);
 342     print "$message\n";
 343     return $is_rebuildable;
 344 }
 345
 346
 347 # Return F:R:Set:Packages than can be rebuilt now
 348 sub select_rebuildable {
 349     my $self = shift;
 350     my @packages = $self->remaining_packages->packages;
 351     my $scheduler = Fedora::Rebuild::Scheduler->new(
 352         limit => $self->selectthreads,
 353         name => 'Selecting buildable packages',
 354         total => $#packages
 355     );
 356     my %jobs = ();
 357     my %finished = ();
 358     print "Selecting rebuildable packages...\n";
 359
 360     my $rebuildables = Fedora::Rebuild::Set::Package->new;
 361     foreach my $package (@packages) {
 362         my $job = $scheduler->schedule($self->can('is_rebuildable'), $self,
 363             $package);
 364         if (! defined $job) { next; }
 365         $jobs{$job} = $package;
 366         %finished = (%finished, ($scheduler->finished()));
 367     }
 368     %finished = (%finished, ($scheduler->finish()));
 369
 370     for my $job (keys %finished) {
 371         my $job_status = $finished{$job};
 372         my $package = $jobs{$job};
 373
 374         if ($$job_status[0]) {
 375             $rebuildables->insert($package);
 376         } elsif (!defined $job_status) {
 377             # Could not decide whether the $package is rebuildable. This is
 378             # fatal for the package. Move it to failed packages.
 379             $self->remaining_packages->delete($package);
 380             $self->mark_failed($package);
 381         }
 382     }
 383
 384     print "Packages selected to rebuild (" . $rebuildables->size .
 385         "): " . $rebuildables->string . "\n";
 386     return $rebuildables;
 387 }
 388
 389
 390 # Rebuild all remaining packages
 391 sub run {
 392     my $self = shift;
 393     if (defined $self->repository) {
 394         print "Starting repository HTTP server...\n";
 395         my $url = $self->repository->start;
 396         print "Repository URL is: <$url>\n";
 397     }
 398
 399     print "remaining_packages: " . $self->remaining_packages->string . "\n";
 400     print "done_packages: " . $self->done_packages->string . "\n";
 401     print "Rebuild mode: " . $self->mode .
 402         " " . (($self->ordered)?"ordered":"unordered") . "\n";
 403
 404     if ($self->ordered) {
 405         $self->load_sourcedependencies;
 406         $self->load_binarydependencies;
 407     }
 408
 409     while ($self->remaining_packages->size > 0) {
 410         my $rebuildable_packages = $self->select_rebuildable;
 411         if ($rebuildable_packages->size == 0) {
 412             printf "No more packages can be rebuilt!\n";
 413             last;
 414         }
 415
 416         my $scheduler = Fedora::Rebuild::Scheduler->new(
 417             limit => $self->threads,
 418             name => 'Building',
 419             total => $rebuildable_packages->size
 420         );
 421
 422         # Rebuild packages
 423         my %jobs = ();
 424         my @packages = $rebuildable_packages->packages;
 425         my $i = 0;
 426
 427         foreach my $package (@packages) {
 428             $self->remaining_packages->delete($package);
 429
 430             my $job = $scheduler->schedule($package->can('rebuild'), $package,
 431                 $self->mode, []);
 432             if (! defined $job) { next; }
 433             $jobs{$job} = $package;
 434             my %finished = $scheduler->finish(++$i < @packages);
 435
 436             while (my ($job, $status) = each %finished) {
 437                 my $package = $jobs{$job};
 438                 if ($$status[0]) {
 439                     # TODO: Push Provides into global list of available
 440                     # provides.
 441                     # XXX: Nothing here, fall through to rotation waiting,
 442                     # rebuilt package list tracked in $rebuildable_packages.
 443                 } else {
 444                     $rebuildable_packages->delete($package);
 445                     $self->mark_failed($package);
 446                 }
 447             }
 448         }
 449
 450
 451         # Wait for Koji rotation
 452         # This is separeted from rebuilding process to solve trade-off between
 453         # high threads number (consumes lot of memory and CPU time when
 454         # starting) and the rebuild itself is much faster Koji rotation period
 455         # (lot of time lost to wait for rotation).
 456         %jobs = ();
 457         @packages = $rebuildable_packages->packages;
 458         $i = 0;
 459
 460         $scheduler = Fedora::Rebuild::Scheduler->new(
 461             limit => $self->threads,
 462             name => 'Waiting for build root',
 463             total => $rebuildable_packages->size
 464         );
 465
 466         foreach my $package (@packages) {
 467             my $job = $scheduler->schedule($package->can('waitforbuildroot'),
 468                 $package, $self->mode);
 469             if (! defined $job) { next; }
 470             $jobs{$job} = $package;
 471             my %finished = $scheduler->finish(++$i < @packages);
 472
 473             while (my ($job, $status) = each %finished) {
 474                 my $package = $jobs{$job};
 475                 if ($$status[0]) {
 476                     $self->repository->insert($package);
 477                     $self->mark_done($package);
 478                 } else {
 479                     $self->mark_failed($package);
 480                 }
 481             }
 482         }
 483         $self->repository->update;
 484     }
 485
 486     if ($self->failed_packages->size > 0) {
 487         print "Rebuild of following packages failed (" .
 488             $self->failed_packages->size . "): " .
 489             $self->failed_packages->string . "\n";
 490         return 0;
 491     } else {
 492         print "All packages have been rebuilt.\n";
 493         return 1;
 494     }
 495 }
 496
 497 1;
 498 __END__
 499 =encoding utf8
 500
 501 =head1 NAME
 502
 503 Fedora::Rebuild - Rebuilds Fedora packages from scratch
 504
 505 =head1 DESCRIPTION
 506
 507 Main goal is to rebuild perl modules packages for Fedora. The rebuild is
 508 driven from bottom to top, i.e. from perl interpreter to modules depending on
 509 intermediate modules. This way, it's possible to upgrade perl interpreter to
 510 incompatible version and to rebuild all modules against the new interpreter.
 511
 512 Procedure is following: We have a set of source package to rebuild. We
 513 distill all build-time dependencies for each source package. We ignore
 514 non-perl dependencies as we focus on Perl only.
 515
 516 We select all source packages that do not depend on anything and push them
 517 into Koji to rebuild. Obviously, the only fulfilling package is Perl interpret
 518 only.
 519
 520 Once the first subset of packages is rebuilt, we gather their binary packages
 521 and distill their binary provides. These freshly available provides are put
 522 into new set of available provides. At the same time we remove the rebuilt
 523 subset from original set of all source packages.
 524
 525 Then we wait for build root rotation in Koji to get rebuilt binary packages
 526 available in build root.
 527
 528 (Of course we could get provides of new binary packages from koji repository
 529 after rotation and we can get provides through repoquery instead of
 530 inspecting binary packages manually.)
 531
 532 If package rebuild fails, the package is removed from future consideration.
 533
 534 Then we return to start of this procedure to select other subset of source
 535 packages whose build-time dependecies can be satisfied from set of binary
 536 provides we have obtained till now.
 537
 538 This loop cycles until set of source packages is empty.
 539
 540 =head1 SYNCHRONIZATION
 541
 542 Because mass rebuild is long term issue and lot of intermediate failures can
 543 emerge, one must be able to resume failed rebuild process.
 544
 545 Safe resume is assured by proper logging. Once a package has been rebuilt, its
 546 name is logged into list of done packages. Restarting rebuild program with the
 547 same arguments loads done packages, remove them from set of remaining
 548 packages, and populates list of available provides. In additon, packages whose
 549 rebuild failed are removed from set of remaining packages.
 550
 551 =head1 AUTHOR
 552
 553 Petr Písař <ppisar@redhat.com>
 554
 555 =head1 COPYING
 556
 557 Copyright (C) 2011  Petr Pisar <ppisar@redhat.com>
 558
 559 This program is free software: you can redistribute it and/or modify
 560 it under the terms of the GNU General Public License as published by
 561 the Free Software Foundation, either version 3 of the License, or
 562 (at your option) any later version.
 563
 564 This program is distributed in the hope that it will be useful,
 565 but WITHOUT ANY WARRANTY; without even the implied warranty of
 566 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 567 GNU General Public License for more details.
 568
 569 You should have received a copy of the GNU General Public License
 570 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 571
 572 =cut
 573