lib/Fedora/Rebuild.pm

   1 package Fedora::Rebuild;
   2 use strict;
   3 use warnings;
   4 use version 0.77; our $VERSION = version->declare("v0.9.0");
   5
   6 use Moose;
   7 use Moose::Util::TypeConstraints;
   8 use File::Path;
   9 use Carp;
  10 use IO::Handle;
  11 use Fedora::Rebuild::Types qw( Mode );
  12 use Fedora::Rebuild::Execute;
  13 use Fedora::Rebuild::Mock;
  14 use Fedora::Rebuild::Package;
  15 use Fedora::Rebuild::Repository;
  16 use Fedora::Rebuild::Scheduler;
  17 use Fedora::Rebuild::Set::Package;
  18 use Fedora::Rebuild::Solver;
  19 use namespace::clean;
  20
  21 has 'all' => (is => 'ro', isa => 'Str', required => 1);
  22 has 'done' => (is => 'ro', isa => 'Str', required => 1);
  23 has 'failed' => (is => 'ro', isa => 'Str', required => 1);
  24 has 'workdir' => (is => 'ro', isa => 'Str', required => 1);
  25 # Directory where repository with built packages live. This is needed for
  26 # `mock' build mode.
  27 has 'repodir' => (is => 'ro', isa => 'Str', required => 1);
  28 # Reference to array of repository URLs. Usually points to Fedora mirror or
  29 # Koji repository (baseurl in YUM terminology). This is needed for `mock'
  30 # build mode.
  31 has 'repourls' => (is => 'rw', isa => 'ArrayRef[Str]', required => 1);
  32 # RPM architecture name, e.g. x86_64.
  33 # TODO: Get current architecture by rpmGetArchInfo() from librpm
  34 has 'architecture' => (is => 'ro', isa => 'Str', required => 1);
  35 # Clone git repositories anonymously. This will not work with `koji' mode.
  36 # Default is non-anonymous clone which requires preconfigured SSH public key
  37 # authentication.
  38 has 'anonymous' => (is => 'ro', isa => 'Bool', required => 0, default => 0);
  39 # Git branch name
  40 # "f14", "f15" etc. Use "rawhide" for latest one.
  41 has 'dist' => (is => 'ro', isa => 'Str', required => 1);
  42 # Fedpkg dist name identifier. This is required only if git branch name is not
  43 # an offical one (e.g. a private branch).
  44 # "f14", "f15" etc. Use "f18" for the latest one.
  45 has 'pkgdist' => (is => 'ro', isa => 'Maybe[Str]', required => 0,
  46     default => undef);
  47 # Build target name
  48 # "dist-f14", "dist-f15" etc. Use "dist-rawhide" for latest one.
  49 has 'target' => (is => 'ro', isa => 'Str', required => 1);
  50 has 'message' => (is => 'ro', isa => 'Str', required => 1);
  51 # Reference to function with three arguments (BuildRequire name, relation
  52 # flag, version) returning false the BuildRequire should be considered, true
  53 # otherwise. If attribute is undef or missing, no filtering will be performed
  54 # (i.e. the same effect as sub {1}).
  55 has 'buildrequiresfilter' => (is => 'ro', isa => 'CodeRef', required => 0);
  56 # Pass "local" committing and building locally only, pass "koji" for pushing
  57 # commits and building in Koji. Default is "koji" to build in Koji.
  58 has 'mode' => (is => 'ro', isa => Mode, required => 0, default => 'koji');
  59 # Run rebuild in given number of threads. Default is 1.
  60 has 'threads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  61         message {"Attribute threads must be positive integer (was $_)"}),
  62     required => 0);
  63 # Load binary provides of already done packages or buildrequires in given
  64 # number of threads. This is to lower local I/O load. Default is 1.
  65 has 'loadthreads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  66         message {"Attribute loadthreads must be positive integer (was $_)"}),
  67     required => 0);
  68 # Select rebuildable packages in given number of threads.
  69 # Dependency solver is CPU intentsive work.
  70 # Default is 1.
  71 has 'selectthreads' => (is => 'ro', isa => subtype('Int' => where {$_>0} =>
  72         message {"Attribute selectthreads must be positive integer (was $_)"}),
  73     required => 0);
  74 # Maximal count of immediately sequential failures to accept and continue
  75 # rebuild process. If the limit exceeds, rebuild process will terminate. This
  76 # is good to catch pathologic cases when something is obviously wrong and
  77 # should be fixed before rebuilding (e.g. Koji is down, or you have not SSH
  78 # key loaded into SSH agent). Use non-positive number to disable this check.
  79 # Default value is 0 (i.e. not to check).
  80 has 'failurelimit' => (is => 'ro', isa => 'Int', required => 0, default => 0);
  81 # Build packages in dependency order. Default is true.
  82 # If set to false, the packages will be built irrespective to dependecies
  83 # (build- and run-time).
  84 has 'ordered' => (is => 'ro', isa => 'Bool', required => 0, default => 1);
  85
  86 has 'remaining_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  87     lazy_build => 1, init_arg => undef);
  88 has 'done_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  89     lazy_build => 1, init_arg => undef);
  90 has 'failed_packages' => (is => 'ro', isa => 'Fedora::Rebuild::Set::Package',
  91     lazy_build => 1, init_arg => undef);
  92 has 'subsequent_failures' => (is => 'rw', isa => 'Int', default => 0,
  93     init_arg => undef);
  94 has 'last_failed' => (is => 'rw', isa => 'Bool', default => 0,
  95     init_arg => undef);
  96 has 'repository' => (is => 'ro', isa => 'Fedora::Rebuild::Repository',
  97     lazy_build => 1, init_arg => undef);
  98
  99
 100 # Creates set of packages already rebuilt.
 101 sub _build_done_packages {
 102     my $self = shift;
 103     my $packages = Fedora::Rebuild::Set::Package->new();
 104     my $file;
 105
 106     if (not -e $self->done) {
 107         print "No packages have been rebuilt yet.\n";
 108         return $packages;
 109     }
 110
 111     open($file, '<', $self->done) or
 112         croak "Could not open `" . $self->done .
 113             "' for loading list of already rebuilt packages: $!";
 114     print "Loading list of already rebuilt package names...\n";
 115     while (local $_ = <$file>) {
 116         chomp;
 117         if (m/^\s*$/) { next; }
 118
 119         my $package = Fedora::Rebuild::Package->new(name => $_,
 120             workdir => $self->workdir, dist => $self->dist,
 121             pkgdist => $self->pkgdist, target => $self->target,
 122             message => $self->message);
 123         $packages->insert($package);
 124         $self->repository->insert($package);
 125     }
 126     if ($!) {
 127         croak "Could not read list of rebuilt package names from file `" .
 128         $self->done . "': $!";
 129     }
 130     close $file;
 131     $self->repository->update;
 132     print "Number of done packages: " . $packages->size() . "\n";
 133
 134     return $packages;
 135 }
 136
 137
 138 sub mark_done {
 139     my ($self, $package) = @_;
 140     my $file = IO::Handle->new();
 141
 142     open($file, '>>', $self->done) or
 143         croak "Could not open `" . $self->done . "' for appending: $!";
 144     printf $file $package->name . "\n" or
 145         croak "Could not write data into `" . $self->done . "': $!";
 146     $file->flush && $file->sync && close($file) or
 147         croak "Could not flush and close `" . $self->done . "': $!";
 148
 149     $self->done_packages->insert($package);
 150     $self->last_failed(0);
 151     $self->subsequent_failures(0);
 152 }
 153
 154 # Creates set of packages not yet rebuilt not already failed.
 155 sub _build_remaining_packages {
 156     my $self = shift;
 157     my $packages = Fedora::Rebuild::Set::Package->new();
 158     my $file;
 159
 160     open($file, '<', $self->all) or
 161         croak "Could not open " . $self->all .
 162             " for loading list of package names to rebuild: $!";
 163     print "Loading list of all package names to rebuild...\n";
 164
 165     while (local $_ = <$file>) {
 166         chomp;
 167         if (m/^\s*$/) { next; }
 168         if ($packages->contains($_)) { next; }
 169         if (! $self->done_packages->contains($_) &&
 170                 ! $self->failed_packages->contains($_)) {
 171             $packages->insert(Fedora::Rebuild::Package->new(name => $_,
 172                 workdir => $self->workdir, dist => $self->dist,
 173                 pkgdist => $self->pkgdist, target => $self->target,
 174                 message => $self->message));
 175         }
 176     }
 177     if ($!) {
 178         croak "Could not read list of package names to rebuild from file `" .
 179             $self->all . "' :$!";
 180     }
 181     close $file;
 182
 183     print "Number of all packages: " .
 184         ($packages->size + $self->done_packages->size
 185             + $self->failed_packages->size) . "\n";
 186     print "Number of not yet rebuilt packages: " . $packages->size() . "\n";
 187     return $packages;
 188 }
 189
 190
 191 # Set of packages whose rebuild failed.
 192 sub _build_failed_packages {
 193     my $self = shift;
 194     my $packages = Fedora::Rebuild::Set::Package->new();
 195     my $file;
 196
 197     if (not -e $self->failed) {
 198         print "No packages have failed yet.\n";
 199         return $packages;
 200     }
 201
 202     open($file, '<', $self->failed) or
 203         croak "Could not open `" . $self->failed .
 204             "' for loading list of already failed packages: $!";
 205     print "Loading list of already failed package names...\n";
 206     while (local $_ = <$file>) {
 207         chomp;
 208         if (m/^\s*$/) { next; }
 209
 210         my $package = Fedora::Rebuild::Package->new(name => $_,
 211             workdir => $self->workdir, dist => $self->dist,
 212             pkgdist => $self->pkgdist, target => $self->target,
 213             message => $self->message);
 214         $packages->insert($package);
 215     }
 216     if ($!) {
 217         croak "Could not read list of failed package names from file `" .
 218         $self->failed . "': $!";
 219     }
 220     close $file;
 221     print "Number of failed packages: " . $packages->size() . "\n";
 222
 223     return $packages;
 224 }
 225
 226 sub _build_repository {
 227     my $self = shift;
 228     return Fedora::Rebuild::Repository->new(path => $self->repodir);
 229 }
 230
 231 # Record package names into log of failed packages
 232 sub mark_failed {
 233     my ($self, $package) = @_;
 234     my $file = IO::Handle->new();
 235
 236     # Log failure
 237     open($file, '>>', $self->failed) or
 238         croak "Could not open `" . $self->failed . "' for appending: $!";
 239     printf $file $package->name . "\n" or
 240         croak "Could not write data into `" . $self->failed . "': $!";
 241     $file->flush && $file->sync && close($file) or
 242         croak "Could not flush and close `" . $self->failed . "': $!";
 243
 244     # Move to list of failed
 245     $self->failed_packages->insert($package);
 246
 247     # Check for very failures
 248     if ($self->last_failed) {
 249         $self->subsequent_failures($self->subsequent_failures + 1);
 250     } else {
 251         $self->last_failed(1);
 252         $self->subsequent_failures(1);
 253     }
 254     if ($self->failurelimit > 0 &&
 255             $self->subsequent_failures > $self->failurelimit) {
 256         croak "More then " . $self->failurelimit .
 257             " package(s) failed subsequently which is more then set " .
 258             "threshold. Aborting now.\n";
 259     }
 260 }
 261
 262 # Load build-requires for each not-yet done package.
 263 # Return true in case of success or croaks in case of failure.
 264 sub load_sourcedependencies {
 265     my $self = shift;
 266     my @packages = $self->remaining_packages->packages;
 267     my $scheduler = Fedora::Rebuild::Scheduler->new(
 268         limit => $self->loadthreads,
 269         name => 'Loading build-requires',
 270         total => $#packages
 271     );
 272     my %jobs = ();
 273     my $i = 0;
 274
 275     my $mock;
 276     if ($self->loadthreads == 1) {
 277         print "Initializing shared mock environment...\n";
 278         eval {
 279             $mock = Fedora::Rebuild::Mock->new(
 280                     architecture => $self->architecture,
 281                     repositories => $self->repourls,
 282                     shared => 1);
 283             $mock->make_configuration;
 284         };
 285         if ($@) {
 286             File::Path::remove_tree($mock->config_dir);
 287             croak("Could not configure mock: $@\n");
 288         }
 289         if (!Fedora::Rebuild::Execute->new->do(undef, 'mock',
 290                 '--configdir', $mock->config_dir, '--root', $mock->config_root,
 291                 '--no-cleanup-after', '--init')) {
 292             File::Path::remove_tree($mock->config_dir);
 293             croak("Could not initilize shared mock environment " .
 294                 "for building source packages.\n");
 295         }
 296         print "Shared mock environment initialized successfully.\n";
 297     }
 298
 299     print "Loading build-time dependenices of not yet rebuilt packages...\n";
 300
 301     foreach my $package (@packages) {
 302         my $job = $scheduler->schedule($package->can('get_buildrequires'),
 303             $package, $self->mode, $self->repourls, $self->architecture,
 304             $self->anonymous, $mock);
 305         if (! defined $job) { next; }
 306         $jobs{$job} = $package;
 307         my %finished = $scheduler->finish(++$i < @packages);
 308
 309         while (my ($job, $status) = each %finished) {
 310             my $package = $jobs{$job};
 311             if (!$$status[0]) {
 312                 print "Could not load build-time dependencies for not yet " .
 313                     "built package `" . $package->name . "'.\n";
 314                 print "Waiting for finishing scheduled jobs...\n";
 315                 $scheduler->finish(1);
 316                 print "All jobs loading build-time dependcies have finished.\n";
 317                 if (defined $mock) {
 318                     File::Path::remove_tree($mock->config_dir);
 319                 }
 320                 croak "Could not load build-time dependencies.\n";
 321             }
 322         }
 323     }
 324
 325     if (defined $mock) {
 326         File::Path::remove_tree($mock->config_dir);
 327     }
 328     print "Build-time dependencies of not-yet rebuilt packages loaded " .
 329         "successfully.\n";
 330     return 1;
 331 }
 332
 333
 334 # Load binary requires and provides of each done package.
 335 # Return true in case of success or croaks in case of failure.
 336 sub load_binarydependencies {
 337     my $self = shift;
 338     my @packages = $self->done_packages->packages;
 339     my $scheduler = Fedora::Rebuild::Scheduler->new(
 340         limit => $self->loadthreads,
 341         name => 'Loading binary dependendies',
 342         total => $#packages
 343     );
 344     my %jobs = ();
 345     my $i = 0;
 346
 347     print "Loading binary dependenices of already rebuilt packages...\n";
 348
 349     foreach my $package (@packages) {
 350         my $job = $scheduler->schedule($package->can('get_binarydependencies'),
 351             $package, $self->mode);
 352         if (! defined $job) { next; }
 353         $jobs{$job} = $package;
 354         my %finished = $scheduler->finish(++$i < @packages);
 355
 356         while (my ($job, $status) = each %finished) {
 357             my $package = $jobs{$job};
 358             if (!$$status[0]) {
 359                 print "Could not load binary dependencies for already built " .
 360                     "package `" . $package->name . "'.\n";
 361                 print "Waiting for finishing scheduled jobs...\n";
 362                 $scheduler->finish(1);
 363                 print "All jobs loading binary dependcies have finished.\n";
 364                 croak "Could not load binary dependencies\n";
 365             }
 366         }
 367     }
 368
 369     print "Binary dependencies of already rebuilt packages loaded " .
 370         "successfully.\n";
 371     return 1;
 372 }
 373
 374
 375 # Decides a package is rebuildable now.
 376 # Return 0 for false, 1 or true, undef for error while deciding.
 377 sub is_rebuildable {
 378     my ($self, $package) = @_;
 379     my $is_rebuildable;
 380     my $message = '';
 381
 382     if ($self->ordered) {
 383         my $solver = Fedora::Rebuild::Solver->new(
 384             'packages' => $self->done_packages,
 385             'dependencyfilter' => $self->buildrequiresfilter);
 386
 387         $is_rebuildable = $solver->is_buildable($package, \$message);
 388     } else {
 389         $is_rebuildable = 1;
 390         $message = "Package `" . $package->name .
 391             "' is buildable because unordered build mode is selected.";
 392     }
 393
 394     $package->log_is_rebuildable($is_rebuildable, $message);
 395     print "$message\n";
 396     return $is_rebuildable;
 397 }
 398
 399
 400 # Return F:R:Set:Packages than can be rebuilt now
 401 sub select_rebuildable {
 402     my $self = shift;
 403     my @packages = $self->remaining_packages->packages;
 404     my $scheduler = Fedora::Rebuild::Scheduler->new(
 405         limit => $self->selectthreads,
 406         name => 'Selecting buildable packages',
 407         total => $#packages
 408     );
 409     my %jobs = ();
 410     my %finished = ();
 411     print "Selecting rebuildable packages...\n";
 412
 413     my $rebuildables = Fedora::Rebuild::Set::Package->new;
 414     foreach my $package (@packages) {
 415         my $job = $scheduler->schedule($self->can('is_rebuildable'), $self,
 416             $package);
 417         if (! defined $job) { next; }
 418         $jobs{$job} = $package;
 419         %finished = (%finished, ($scheduler->finished()));
 420     }
 421     %finished = (%finished, ($scheduler->finish()));
 422
 423     for my $job (keys %finished) {
 424         my $job_status = $finished{$job};
 425         my $package = $jobs{$job};
 426
 427         if ($$job_status[0]) {
 428             $rebuildables->insert($package);
 429         } elsif (!defined $job_status) {
 430             # Could not decide whether the $package is rebuildable. This is
 431             # fatal for the package. Move it to failed packages.
 432             $self->remaining_packages->delete($package);
 433             $self->mark_failed($package);
 434         }
 435     }
 436
 437     print "Packages selected to rebuild (" . $rebuildables->size .
 438         "): " . $rebuildables->string . "\n";
 439     return $rebuildables;
 440 }
 441
 442
 443 # Rebuild all remaining packages
 444 sub run {
 445     my $self = shift;
 446     if (defined $self->repository) {
 447         if (! defined $self->repository->url) {
 448             print "Starting repository HTTP server...\n";
 449             $self->repository->start;
 450             push @{$self->repourls}, $self->repository->url;
 451         }
 452         print "Repository URL is: <" . $self->repository->url . ">\n";
 453     }
 454
 455     print "remaining_packages: " . $self->remaining_packages->string . "\n";
 456     print "done_packages: " . $self->done_packages->string . "\n";
 457     print "Rebuild mode: " . $self->mode .
 458         " " . (($self->ordered)?"ordered":"unordered") . "\n";
 459
 460     if ($self->ordered) {
 461         $self->load_sourcedependencies;
 462         $self->load_binarydependencies;
 463     }
 464
 465     while ($self->remaining_packages->size > 0) {
 466         my $rebuildable_packages = $self->select_rebuildable;
 467         if ($rebuildable_packages->size == 0) {
 468             printf "No more packages can be rebuilt!\n";
 469             last;
 470         }
 471
 472         my $scheduler = Fedora::Rebuild::Scheduler->new(
 473             limit => $self->threads,
 474             name => 'Building',
 475             total => $rebuildable_packages->size
 476         );
 477
 478         # Rebuild packages
 479         my %jobs = ();
 480         my @packages = $rebuildable_packages->packages;
 481         my $i = 0;
 482
 483         foreach my $package (@packages) {
 484             $self->remaining_packages->delete($package);
 485
 486             my $job = $scheduler->schedule($package->can('rebuild'), $package,
 487                 $self->mode, $self->repourls, $self->architecture,
 488                 $self->anonymous);
 489             if (! defined $job) { next; }
 490             $jobs{$job} = $package;
 491             my %finished = $scheduler->finish(++$i < @packages);
 492
 493             while (my ($job, $status) = each %finished) {
 494                 my $package = $jobs{$job};
 495                 if ($$status[0]) {
 496                     # TODO: Push Provides into global list of available
 497                     # provides.
 498                     # XXX: Nothing here, fall through to rotation waiting,
 499                     # rebuilt package list tracked in $rebuildable_packages.
 500                 } else {
 501                     $rebuildable_packages->delete($package);
 502                     $self->mark_failed($package);
 503                 }
 504             }
 505         }
 506
 507
 508         # Wait for Koji rotation
 509         # This is separeted from rebuilding process to solve trade-off between
 510         # high threads number (consumes lot of memory and CPU time when
 511         # starting) and the rebuild itself is much faster Koji rotation period
 512         # (lot of time lost to wait for rotation).
 513         %jobs = ();
 514         @packages = $rebuildable_packages->packages;
 515         $i = 0;
 516
 517         $scheduler = Fedora::Rebuild::Scheduler->new(
 518             limit => $self->threads,
 519             name => 'Waiting for build root',
 520             total => $rebuildable_packages->size
 521         );
 522
 523         foreach my $package (@packages) {
 524             my $job = $scheduler->schedule($package->can('waitforbuildroot'),
 525                 $package, $self->mode);
 526             if (! defined $job) { next; }
 527             $jobs{$job} = $package;
 528             my %finished = $scheduler->finish(++$i < @packages);
 529
 530             while (my ($job, $status) = each %finished) {
 531                 my $package = $jobs{$job};
 532                 if ($$status[0]) {
 533                     $self->repository->insert($package);
 534                     $self->mark_done($package);
 535                 } else {
 536                     $self->mark_failed($package);
 537                 }
 538             }
 539         }
 540         $self->repository->update;
 541     }
 542
 543     if ($self->remaining_packages->size > 0) {
 544         print "Following packages have unsatisfied build-time dependencies (" .
 545         $self->remaining_packages->size . "): " .
 546         $self->remaining_packages->string . "\n";
 547     }
 548     if ($self->failed_packages->size > 0) {
 549         print "Following packages have failed while rebuilding (" .
 550             $self->failed_packages->size . "): " .
 551             $self->failed_packages->string . "\n";
 552         return 0;
 553     } else {
 554         print "All rebuildable packages have been rebuilt.\n";
 555         return 1;
 556     }
 557 }
 558
 559 1;
 560 __END__
 561 =encoding utf8
 562
 563 =head1 NAME
 564
 565 Fedora::Rebuild - Rebuilds Fedora packages from scratch
 566
 567 =head1 DESCRIPTION
 568
 569 Main goal is to rebuild perl modules packages for Fedora. The rebuild is
 570 driven from bottom to top, i.e. from perl interpreter to modules depending on
 571 intermediate modules. This way, it's possible to upgrade perl interpreter to
 572 incompatible version and to rebuild all modules against the new interpreter.
 573
 574 Procedure is following: We have a set of source package to rebuild. We
 575 distill all build-time dependencies for each source package. We ignore
 576 non-perl dependencies as we focus on Perl only.
 577
 578 We select all source packages that do not depend on anything and push them
 579 into Koji to rebuild. Obviously, the only fulfilling package is Perl interpret
 580 only.
 581
 582 Once the first subset of packages is rebuilt, we gather their binary packages
 583 and distill their binary provides. These freshly available provides are put
 584 into new set of available provides. At the same time we remove the rebuilt
 585 subset from original set of all source packages.
 586
 587 Then we wait for build root rotation in Koji to get rebuilt binary packages
 588 available in build root.
 589
 590 (Of course we could get provides of new binary packages from koji repository
 591 after rotation and we can get provides through repoquery instead of
 592 inspecting binary packages manually.)
 593
 594 If package rebuild fails, the package is removed from future consideration.
 595
 596 Then we return to start of this procedure to select other subset of source
 597 packages whose build-time dependecies can be satisfied from set of binary
 598 provides we have obtained till now.
 599
 600 This loop cycles until set of source packages is empty.
 601
 602 =head1 SYNCHRONIZATION
 603
 604 Because mass rebuild is long term issue and lot of intermediate failures can
 605 emerge, one must be able to resume failed rebuild process.
 606
 607 Safe resume is assured by proper logging. Once a package has been rebuilt, its
 608 name is logged into list of done packages. Restarting rebuild program with the
 609 same arguments loads done packages, remove them from set of remaining
 610 packages, and populates list of available provides. In additon, packages whose
 611 rebuild failed are removed from set of remaining packages.
 612
 613 =head1 AUTHOR
 614
 615 Petr Písař <ppisar@redhat.com>
 616
 617 =head1 COPYING
 618
 619 Copyright (C) 2011, 2012  Petr Pisar <ppisar@redhat.com>
 620
 621 This program is free software: you can redistribute it and/or modify
 622 it under the terms of the GNU General Public License as published by
 623 the Free Software Foundation, either version 3 of the License, or
 624 (at your option) any later version.
 625
 626 This program is distributed in the hope that it will be useful,
 627 but WITHOUT ANY WARRANTY; without even the implied warranty of
 628 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 629 GNU General Public License for more details.
 630
 631 You should have received a copy of the GNU General Public License
 632 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 633
 634 =cut
 635