tools/build_cluster_controller.pl

   1 #! /usr/bin/perl
   2 # ======================================================================
   3 # ======================================================================
   4
   5 use strict;
   6 use warnings;
   7 use Socket;
   8 use File::Copy;
   9
  10 # ======================================================================
  11 # Constants
  12 # ======================================================================
  13
  14 use constant START_COMMUNICATION => "S";
  15 use constant START_NOT_LOCKED_READY => "B";
  16 use constant START_LOCKED_READY => "C";
  17 use constant START_LOCKED_READY_UNAUTHORIZED_USER => "L";
  18 use constant START_ERROR_AUTHORIZING => "K";
  19
  20 use constant END_COMMUNICATION => "E";
  21 use constant END_SUCCESSFULL => "F";
  22
  23 use constant SUCCESSFULL_COMMAND => "P";
  24 use constant FAILED_COMMAND => "U";
  25 use constant UPDATE_BOOTLEG_STEP_OK => "G";
  26 use constant UPDATE_BOOTLEG_STEP_FAILED => "H";
  27 use constant UPDATE_BOOTLEG_SEND_DIRECTORY => "M";
  28 use constant UPDATE_BOOTLEG_SEND_FILE => "N";
  29 use constant UPDATE_BOOTLEG_FILES_FINISHED => "Q";
  30 use constant SNAPSHOT_FAILED => "O";
  31 use constant SNAPSHOT_SUCCESSFULL => "P";
  32
  33 use constant COMMAND_RESTART => "a";
  34 use constant COMMAND_RESTART_LOGIN => "b";
  35 use constant COMMAND_RESTART_NODES => "c";
  36 use constant COMMAND_LOCK => "d";
  37 use constant COMMAND_UNLOCK => "e";
  38 use constant COMMAND_UPDATE_BOOTLEG => "f";
  39 use constant COMMAND_CONTENT_SYNC => "g";
  40 use constant COMMAND_SYNC_SPECIFIED_CHANGELISTS => "h";
  41 use constant COMMAND_SNAPSHOT => "i";
  42 use constant COMMAND_BOOTLEG_VERSION => "j";
  43 use constant COMMAND_FREE_OBJECT_IDS => "k";
  44
  45 # ======================================================================
  46 # Globals
  47 # ======================================================================
  48
  49 my $buildCluster        = "swo-dev9.station.sony.com";
  50 my $port                = "98452";
  51 my $candela             = "p:";
  52 my $exitcode            = 0;
  53
  54 my $name = $0;
  55 $name =~ s/^(.*)\\//;
  56
  57 my $option;
  58 my $command;
  59 my $user;
  60
  61 # ======================================================================
  62 # Subroutines
  63 # ======================================================================
  64
  65 sub usage
  66 {
  67         print STDERR "\nUsage:\n";
  68         print STDERR "\t$name [command(s)]\n\n".
  69                      "\t\t-restart :\n\t\t\t restart the build cluster (central node)\n".
  70                      "\t\t-restart-login :\n\t\t\t restart the Login server\n".
  71                      "\t\t-restart-nodes :\n\t\t\t restart all nodes of the build cluster\n".
  72                      "\t\t-lock :\n\t\t\t lock the build cluster (must be authorized user)\n".
  73                      "\t\t-unlock :\n\t\t\t unlock the build cluster (must be authorized user)\n".
  74                      "\t\t-update-bootleg <branch> :\n\t\t\t update the bootleg on the build cluster (p4 key check) - needs to be run in windows\n".
  75                      "\t\t-bootleg-version:\n\t\t\t check bootleg version on the build cluster\n".
  76                      "\t\t-free-object-ids :\n\t\t\t free object IDs in the database for the build cluster\n".
  77                      "\t\t-content-sync [changelist] :\n\t\t\t shut down, content sync to specific changelist (if none, content sync to head), bring up\n".
  78                      "\t\t-sync-specified-changelists <changelist [| changelist]> :\n\t\t\t shut down, sync to multiple specified changelists, bring up\n".
  79                      "\t\t-snap <schema> <branch> [dontsubmit] :\n\t\t\t free object IDs, make a snapshot, verifies before adding files to <branch> in perforce\n\t\t\t  and submits unless [dontsubmit]\n".
  80                          "\t\t\t If <schema> does not exist, it is created otherwise it is overwritten\n".
  81                      "\n\tIf multiple commands are given, the build cluster will go down / come up only once around the commands (if necessary)\n";
  82         die "\n";
  83 }
  84
  85 sub exitFailed
  86 {
  87         $exitcode = 1;
  88         goto FAIL;
  89 }
  90
  91 sub perforceWhere
  92 {
  93         local $_;
  94
  95         # find out where a perforce file resides on the local machine
  96         my $result;
  97         {
  98                 open(P4, "p4 where $_[0] |");
  99                         $_ = <P4>;
 100                         chomp;
 101                         my @where = split;
 102                         $result = $where[2];
 103                 close(P4);
 104         }
 105
 106         return $result;
 107 }
 108
 109 sub checkarguments()
 110 {
 111         my @args = @ARGV;
 112
 113         while(@args)
 114         {
 115                 my $elem = shift @args;
 116
 117                 # check if the key is valid if the command requires one
 118                 if($elem =~ /^-snap$/)
 119                 {
 120                         $elem = shift @args;
 121                         &usage() if(!(defined $elem) || $elem =~ /^-/);
 122                         $elem = shift @args;
 123                         &usage() if(!(defined $elem) || $elem =~ /^-/);
 124                         # check for optional parameter
 125                         shift @args if((defined $args[0]) && $args[0] eq "dontsubmit");
 126                 }
 127                 elsif($elem =~ /^-update-bootleg$/)
 128                 {
 129                         $elem = shift @args;
 130                         &usage() if(!(defined $elem) || $elem =~ /^-/);
 131                 }
 132                 elsif($elem =~ /^-content-sync$/)
 133                 {
 134                         shift @args if(@args && !($args[0] =~ /^-/));
 135                 }
 136                 elsif($elem =~ /^-sync-specified-changelists$/)
 137                 {
 138                         $elem = shift @args;
 139                         &usage() if(!defined $elem || $elem =~ /^-/);
 140                         while(@args)
 141                         {
 142                                 last if($args[0] =~ /^-/);
 143                                 shift @args;
 144                         }
 145                 }
 146                 elsif(!($elem =~ /^-restart$/ || $elem =~ /^-restart-login$/ || $elem =~ /^-restart-nodes$/ || $elem =~ /^-lock$/ || $elem =~ /^-unlock$/ || $elem =~ /^-bootleg-version$/ || $elem =~ /^-free-object-ids$/ || $elem =~ /^-build_script_publish$/))
 147                 {
 148                         &usage();
 149                 }
 150         }
 151 }
 152
 153 sub openbuildsocket
 154 {
 155         socket(BUILD, PF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket failed\n";
 156         {
 157                 my $destination = inet_aton($buildCluster) || die "inet_aton failed\n";
 158                 my $paddr = sockaddr_in($port, $destination);
 159                 connect(BUILD, $paddr) || die "connect failed\n";
 160
 161                 # unbuffer the socket
 162                 my $oldSelect = select(BUILD);
 163                 $| = 1;
 164                 select($oldSelect);
 165
 166                 # put the socket into binary mode
 167                 binmode BUILD;
 168         }
 169 }
 170
 171 sub getuser
 172 {
 173         my $user;
 174
 175         open(P4, "p4 user -o |") || die "p4 user failed\n";
 176         while(<P4>)
 177         {
 178                 $user = $1 if(/^User:\s+(\S+)/);
 179         }
 180         close(P4);
 181
 182         die "Could not get perforce user\n" if(!defined $user);
 183
 184         return $user;
 185 }
 186
 187 sub sendstartinfo
 188 {
 189         print STDERR "Contacting build cluster...\n";
 190         print BUILD START_COMMUNICATION;
 191
 192         my $initializer = $user;
 193         $initializer = "buildscript" if($user eq "build_script_publish");
 194
 195         my $length = length $initializer;
 196
 197         print BUILD pack("N", $length);
 198         print BUILD $initializer;
 199
 200         my $returncode;
 201         if(read(BUILD, $returncode, 1) != 1)
 202         {
 203                 print STDERR "Problem contacting build server\n";
 204                 return 0;
 205         }
 206
 207         if($returncode eq START_NOT_LOCKED_READY)
 208         {
 209                 print STDERR "Build server is not locked and ready\n\n";
 210                 return 1;
 211         }
 212         elsif($returncode eq START_LOCKED_READY)
 213         {
 214                 print STDERR "Build server is locked and ready\n\n";
 215                 return 1;
 216         }
 217         elsif($returncode eq START_LOCKED_READY_UNAUTHORIZED_USER)
 218         {
 219                 print STDERR "Build server is locked (limited access for non-authoritative user)\n\n";
 220                 return 1;
 221         }
 222         elsif($returncode eq START_ERROR_AUTHORIZING)
 223         {
 224                 print STDERR "problem authorizing $user for build server\n\n";
 225                 return 0;
 226         }
 227         else
 228         {
 229                 print STDERR "Build server not ready\n\n";
 230                 return 0;
 231         }
 232
 233 }
 234
 235 sub sendendinfo
 236 {
 237         print STDERR "Ending communication with build cluster...\n";
 238         print BUILD END_COMMUNICATION;
 239
 240         my $returncode;
 241         my $readreturn = read(BUILD, $returncode, 1);
 242         if(!defined $readreturn || $readreturn != 1)
 243         {
 244                 print STDERR "Build server communication ended abruptly\n";
 245                 return 0;
 246         }
 247
 248         if($returncode eq END_SUCCESSFULL)
 249         {
 250                 print STDERR "Build server communication ended successfully\n";
 251                 return 1;
 252         }
 253         else
 254         {
 255                 print STDERR "Build server communication ended with errors\n";
 256                 return 0;
 257         }
 258
 259 }
 260
 261 sub contentsync
 262 {
 263         my $changelist = "";
 264
 265         $changelist = shift @ARGV if(@ARGV && !($ARGV[0] =~ /^-/));
 266
 267         my $length = length $changelist;
 268
 269         print BUILD pack("N", $length);
 270         print BUILD $changelist;
 271
 272         # Recieve any errors from the content sync
 273         my $buffer;
 274         return 0 if(read(BUILD, $buffer, 4) != 4);
 275         $length = unpack("N", $buffer);
 276         return 0 if(read(BUILD, $buffer, $length) != $length);
 277         print $buffer;
 278
 279         return 1;
 280 }
 281
 282 sub syncspecifiedchangelists
 283 {
 284         my $changelists = "";
 285
 286         while(@ARGV)
 287         {
 288                 last if($ARGV[0] =~ /^-/);
 289
 290                 my $elem = shift @ARGV;
 291                 $changelists .= "$elem ";
 292         }
 293
 294         chomp $changelists;
 295
 296         if($changelists eq "")
 297         {
 298                 print BUILD pack("N", 0);
 299                 print STDERR "You must specify changelist\(s\)\n";
 300                 return 0;
 301         }
 302
 303         my $length = length $changelists;
 304
 305         print BUILD pack("N", $length);
 306         print BUILD $changelists;
 307
 308         return 1;
 309 }
 310
 311 sub endsubmit
 312 {
 313         print "Error running: $_[0]\n";
 314         return 0;
 315 }
 316
 317 sub submitopenfiles
 318 {
 319         my $dontsubmit = shift;
 320         local $_;
 321         my @files;
 322
 323         system("p4 revert -a > /dev/null");
 324
 325         open(P4, "p4 -ztag opened -c default |");
 326
 327                 while (<P4>)
 328                 {
 329                         chomp;
 330                         push (@files, $_) if (s/^\.\.\. depotFile //);
 331                 }
 332
 333         close(P4);
 334
 335         if(!@files)
 336         {
 337                 print STDERR "No changed files, nothing to submit\n";
 338                 return 1;
 339         }
 340
 341         my $tmpfile = "submit.tmp";
 342
 343         # submit all the open files
 344         open(TMP, ">" . $tmpfile);
 345
 346                 print TMP "Change:\tnew\n";
 347                 print TMP "\nDescription:\n";
 348
 349                 foreach (@_)
 350                 {
 351                         print TMP "\t", $_, "\n";
 352                 }
 353
 354                 print TMP "\nFiles:\n";
 355                 foreach (@files)
 356                 {
 357                         print TMP "\t", $_, "\n";
 358                 }
 359
 360         close(TMP);
 361
 362         if ($dontsubmit)
 363         {
 364                 open(P4, "p4 change -i < $tmpfile |") || return 0;
 365                 while(<P4>)
 366                 {
 367                         print STDERR "Successfully created changelist $1\n" if(/Change (\d+) created/);
 368                 }
 369                 close(P4);
 370         }
 371         else
 372         {
 373                 open(P4, "p4 submit -i < $tmpfile |") || return 0;
 374                 while(<P4>)
 375                 {
 376                         print STDERR "Successfully submitted at changelist $1\n" if(/Change (\d+) submitted/);
 377                 }
 378                 close(P4);
 379         }
 380
 381         return 0 if ($? != 0);
 382         unlink($tmpfile);
 383         return 1;
 384 }
 385
 386 sub snapshot
 387 {
 388         my $dbSchema = shift @ARGV;
 389         my $branch = shift @ARGV;
 390         my $dontsubmit = 0;
 391         my $snapshotLog = "";
 392         my $buffer = "";
 393         my $p4operation = "submit";
 394
 395         if (defined($ARGV[0]) && $ARGV[0] eq "dontsubmit")
 396         {
 397                 $dontsubmit = 1;
 398                 $p4operation = "change";
 399                 shift @ARGV;
 400         }
 401
 402         print BUILD pack("N", length $dbSchema);
 403         print BUILD $dbSchema;
 404
 405         if(read(BUILD, $buffer, 1) != 1 || $buffer eq SNAPSHOT_FAILED)
 406         {
 407                 print STDERR "Snapshot not created successfully on the build cluster\n";
 408                 return 0;
 409         }
 410
 411         # Recieve files
 412         my @worldSnapshots;
 413
 414         print STDERR "Snapshot generation complete.\n";
 415         while(1)
 416         {
 417                 return 0 if (read(BUILD, $buffer, 2*4) != 2*4);
 418                 my ($fileSize, $fileNameLength) = unpack("NN", $buffer);
 419
 420                 # check if we are finished
 421                 last if($fileSize == 0 && $fileNameLength == 0);
 422
 423                 my $localFileName;
 424                 return 0 if (read(BUILD, $localFileName, $fileNameLength) != $fileNameLength);
 425
 426                 # first file sent will be the snapshot log
 427                 $snapshotLog = $localFileName if($snapshotLog eq "");
 428
 429                 # add all ws files to the array
 430                 push @worldSnapshots, $localFileName if($localFileName =~ /\.ws$/);
 431
 432                 # receive the binary bits for the file
 433                 print STDERR "Receiving $localFileName ($fileSize bytes)...";
 434                 unlink $localFileName;
 435                 open(F, ">" . $localFileName) || return endbootleginstall("could not open $localFileName for writing");
 436                         binmode(F);
 437                         while ($fileSize)
 438                         {
 439                                 my $readSize = 16 * 1024;
 440                                 $readSize = $fileSize if ($fileSize < $readSize);
 441                                 my $readResult = read(BUILD, $buffer, $readSize);
 442                                 return 0 if (!defined($readResult));
 443                                 return 0 if ($readResult == 0);
 444                                 print F $buffer;
 445                                 $fileSize -= $readResult;
 446                         }
 447                 return 0 if ($fileSize != 0);
 448                 close(F);
 449                 print "done\n";
 450         }
 451
 452         # Echo log to user
 453         print STDERR "--- Start of snapshot log:\n";
 454         system("cat $snapshotLog") == 0 || return 0;
 455         print STDERR "--- End of snapshot log:\n";
 456
 457         # Only verify using STDIN if we are not being called by the build script
 458         if($user ne "build_script_publish")
 459         {
 460                 print STDERR "\nAre the world snapshots ok to do perforce $p4operation? (y/n)\n";
 461                 while(<STDIN>)
 462                 {
 463                         chomp;
 464
 465                         if($_ eq "y" || $_ eq "Y")
 466                         {
 467                                 last;
 468                         }
 469                         elsif($_ eq "n" || $_ eq "N")
 470                         {
 471                                 return 1;
 472                         }
 473                         print STDERR "Please enter \'y\' or \'n\'\n";
 474                 }
 475         }
 476
 477         # If we get here, we have decided to submit
 478         print STDERR "Proceeding with $p4operation\n";
 479
 480         # Get a hash of the current world snapshots in perforce
 481         my %ws;
 482         open(FILES, "p4 files //depot/swg/$branch/data/sku.0/sys.client/built/game/snapshot/... |") || return endsubmit("p4 files");
 483         while(<FILES>)
 484         {
 485                 $ws{$1} = 1 if(/\/([^\/]+\.ws)#/);
 486         }
 487         close(FILES);
 488
 489         # Edit files and move to appropriate directory
 490         system("p4 edit //depot/swg/$branch/data/sku.0/sys.client/built/game/snapshot/...") == 0 || return endsubmit("p4 edit snapshot dir");
 491
 492         foreach(@worldSnapshots)
 493         {
 494                 system("p4 add //depot/swg/$branch/data/sku.0/sys.client/built/game/snapshot/$_") == 0 || return endsubmit("p4 add") if(!exists($ws{$_}));
 495                 copy($_, perforceWhere("//depot/swg/$branch/data/sku.0/sys.client/built/game/snapshot/$_")) || return endsubmit("moving *.ws to snapshot dir");
 496         }
 497
 498         system("p4 edit //depot/swg/$branch/dsrc/sku.0/sys.client/built/game/snapshot/swg_object.txt") == 0 || return endsubmit("p4 edit swg_object.txt");
 499         copy("swg_object.txt", perforceWhere("//depot/swg/$branch/dsrc/sku.0/sys.client/built/game/snapshot/swg_object.txt")) || return endsubmit("moving object file to swg_object.txt");
 500
 501         # create golddata text file
 502         createGoldDataFile($dbSchema, $branch);
 503
 504         submitopenfiles($dontsubmit, "[automated]", "New snapshots for $branch from build_cluster_controller ($dbSchema)") || return endsubmit("p4 $p4operation");
 505
 506         return 1;
 507 }
 508
 509 sub createGoldDataFile
 510 {
 511         my ($dbSchema, $branch) = @_;
 512         my $goldDataFile = perforceWhere("//depot/swg/$branch/src/game/server/database/build/linux/golddata.txt");
 513
 514         system("p4 edit $goldDataFile");
 515
 516         open(GOLDDATA, "> $goldDataFile");
 517         print GOLDDATA "$dbSchema\n";
 518         close GOLDDATA;
 519
 520         system("p4 add $goldDataFile");
 521 }
 522
 523 sub getbootlegversion
 524 {
 525         my $buffer;
 526         return 0 if(read(BUILD, $buffer, 4) != 4);
 527         my $length = unpack("N", $buffer);
 528         return 0 if(read(BUILD, $buffer, $length) != $length);
 529
 530         if($length == 0)
 531         {
 532                 print STDERR "Could not get build cluster bootleg version\n";
 533                 return 0;
 534         }
 535
 536         print STDERR "Build cluster bootleg version is: $buffer\n";
 537         return 1;
 538 }
 539
 540 sub updatebootleg
 541 {
 542         my $branch = shift @ARGV;
 543
 544         # Get the number of the most recent bootleg
 545         my $dir = "$candela/swo/builds/$branch";
 546
 547         my $buffer;
 548         my $change = 0;
 549         opendir DH, $dir or return 0;
 550         foreach (readdir DH)
 551         {
 552                 $change = $_ if(/^\d+$/ && -d ($dir."/".$_) && $_ > $change);
 553         }
 554         closedir DH;
 555         return 0 if(!$change);
 556
 557         print STDERR "Most recent blessed bootleg is: $change\n";
 558
 559         # Send info to build cluster
 560         print STDERR "Syncing build cluster to $change...\n";
 561         print BUILD pack("N", $change);
 562         return 0 if(read(BUILD, $buffer, 1) != 1 || $buffer ne UPDATE_BOOTLEG_STEP_OK);
 563         print STDERR "Sync of build cluster complete.\n";
 564
 565         # Compress the server binaries
 566         my $file = "servers_debug.tar.gz";
 567
 568         print STDERR "Compressing server binaries...\n";
 569         system("tar --create --gzip --directory=$dir/$change/servers_debug --file=/tmp/$file .") == 0 || die "Failed to compress $dir/$change/servers_debug";
 570         print STDERR "Compress server binaries complete.\n";
 571
 572         # Send file to build cluster
 573         die "Can't find server zip file!\n" if (!-s "c:/cygwin/tmp/$file");
 574         my $fileSize = -s "c:/cygwin/tmp/$file";
 575         print STDERR "Sending file $file ($fileSize bytes)\n";
 576         print BUILD pack("NN", $fileSize, length $file);
 577         print BUILD $file;
 578         open(F, "<c:/cygwin/tmp/$file");
 579                 binmode(F);
 580                 while ($fileSize)
 581                 {
 582                         my $buffer;
 583                         my $readSize = 16 * 1024;
 584                         $readSize = $fileSize if ($fileSize < $readSize);
 585                         my $readResult = read(F, $buffer, $readSize);
 586                         die "unexpected end of file" if (!defined($readResult));
 587                         die "did not read what we expected to" if ($readResult != $readSize);
 588                         print BUILD $buffer;
 589                         $fileSize -= $readResult;
 590                 }
 591                 die "copied all the bytes but not at EOF" if (!eof(F));
 592         close(F);
 593
 594         # Cleanup
 595         unlink "c:/cygwin/tmp/$file";
 596
 597         if(read(BUILD, $buffer, 1) != 1 || $buffer ne UPDATE_BOOTLEG_STEP_OK)
 598         {
 599                 print "Failed while sending file.\n";
 600                 closedir DH;
 601                 return 0;
 602         }
 603         print "$file sent.\n";
 604
 605         print STDERR "Updating database on build cluster...\n";
 606         return 0 if(read(BUILD, $buffer, 1) != 1 || $buffer ne UPDATE_BOOTLEG_STEP_OK);
 607         print STDERR "Database update on build cluster complete.\n";
 608
 609         print STDERR "Syncing individual changelists on build cluster...\n";
 610         my @syncChangelists;
 611         open(SYNC, "$candela/SWO/builds/$branch/$change/sync.txt") || return 0;
 612         while(<SYNC>)
 613         {
 614                 chomp;
 615                 push @syncChangelists, $_;
 616         }
 617         close(SYNC);
 618         print BUILD pack("N", length (join(" ", @syncChangelists)));
 619         print BUILD join(" ", @syncChangelists);
 620         return 0 if(read(BUILD, $buffer, 1) != 1 || $buffer ne UPDATE_BOOTLEG_STEP_OK);
 621         print STDERR "Inidividual changelist sync complete.\n";
 622
 623         return 1;
 624
 625 }
 626
 627 # ======================================================================
 628 # Main
 629 # ======================================================================
 630
 631 &usage if(@ARGV == 0);
 632
 633 # Check to see if we're testing
 634 if($ARGV[0] eq "vthakkar-box")
 635 {
 636         shift;
 637         $buildCluster = "lin-vthakkar.station.sony.com";
 638 }
 639
 640 $user = getuser();
 641 $user = "build_script_publish" if(grep("-build_script_publish" eq $_, @ARGV));
 642
 643 checkarguments();
 644
 645 openbuildsocket();
 646
 647 sendstartinfo() || exitFailed();
 648
 649 while(@ARGV)
 650 {
 651
 652         $option = shift @ARGV;
 653
 654         if($option eq "-restart")
 655         {
 656                 print STDERR "Restarting build cluster...\n";
 657                 print BUILD COMMAND_RESTART;
 658         }
 659         elsif($option eq "-restart-login")
 660         {
 661                 print STDERR "Restarting loginserver on build cluster...\n";
 662                 print BUILD COMMAND_RESTART_LOGIN;
 663         }
 664         elsif($option eq "-restart-nodes")
 665         {
 666                 print STDERR "Restarting build cluster nodes...\n";
 667                 print BUILD COMMAND_RESTART_NODES;
 668         }
 669         elsif($option eq "-lock")
 670         {
 671                 print STDERR "Locking build cluster...\n";
 672                 print BUILD COMMAND_LOCK;
 673         }
 674         elsif($option eq "-unlock")
 675         {
 676                 print STDERR "Unlocking build cluster...\n";
 677                 print BUILD COMMAND_UNLOCK;
 678         }
 679         elsif($option eq "-update-bootleg")
 680         {
 681                 print STDERR "Updating bootleg on build cluster...\n";
 682                 print BUILD COMMAND_UPDATE_BOOTLEG;
 683                 updatebootleg() || goto ERROR;
 684         }
 685         elsif($option eq "-content-sync")
 686         {
 687                 print STDERR "Content syncing build cluster...\n";
 688                 print BUILD COMMAND_CONTENT_SYNC;
 689                 contentsync() || goto ERROR;
 690         }
 691         elsif($option eq "-sync-specified-changelists")
 692         {
 693                 print STDERR "Syncing build cluster to changelists...\n";
 694                 print BUILD COMMAND_SYNC_SPECIFIED_CHANGELISTS;
 695                 syncspecifiedchangelists() || goto ERROR;
 696         }
 697         elsif($option eq "-snap")
 698         {
 699                 print STDERR "Creating snapshot on build cluster...\n";
 700                 print BUILD COMMAND_SNAPSHOT;
 701                 snapshot() || goto ERROR;
 702         }
 703         elsif($option eq "-bootleg-version")
 704         {
 705                 print STDERR "Checking bootleg version on build cluster...\n";
 706                 print BUILD COMMAND_BOOTLEG_VERSION;
 707                 getbootlegversion() || goto ERROR;
 708         }
 709         elsif($option eq "-free-object-ids")
 710         {
 711                 print STDERR "Freeing object ids on build cluster...\n";
 712                 print BUILD COMMAND_FREE_OBJECT_IDS;
 713         }
 714         elsif($option eq "-build_script_publish")
 715         {
 716                 next;
 717         }
 718         else
 719         {
 720                 print STDERR "Error: cannot decipher option: $option\n";
 721                 goto FAIL;
 722         }
 723
 724 ERROR:
 725         my $success;
 726         exitFailed() if(!read(BUILD, $success, 1));
 727         if($success eq SUCCESSFULL_COMMAND)
 728         {
 729                 print STDERR "Successfully completed $option\n\n";
 730         }
 731         else
 732         {
 733                 print STDERR "Error encountered while running $option\n\n";
 734                 exitFailed();
 735         }
 736
 737 }
 738
 739 FAIL:
 740 sendendinfo();
 741
 742 close(BUILD);
 743
 744 exit($exitcode);