TAO/orbsvcs/tests/ImplRepo/run_test.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2
   3     & eval 'exec perl -S $0 $argv:q'
   4     if 0;
   5
   6 # -*- perl -*-
   7
   8 ###############################################################################
   9
  10 use strict;
  11 use lib "$ENV{ACE_ROOT}/bin";
  12 use PerlACE::TestTarget;
  13 use File::Copy;
  14
  15 my $debug_level = '0';
  16 my $srv_debug_level = '0';
  17 my $test_debug_level = '2';
  18 my $num_srvr = 1;
  19 my $replica = 0;
  20 my $all_tests = 0;
  21
  22 foreach my $i (@ARGV) {
  23     if ($i eq '-debug') {
  24         $debug_level = '10';
  25         $srv_debug_level = '10';
  26     }
  27     elsif ($i eq '-tdebug') {
  28         $test_debug_level = 10;
  29     }
  30     elsif ($i eq '-tsilent') {
  31         $test_debug_level = 0;
  32     }
  33     elsif ($i eq '-servers') {
  34       $num_srvr = 3;
  35     }
  36     elsif ($i eq '-replica') {
  37       $num_srvr = 2;
  38       $replica = 1;
  39     }
  40     elsif ($i eq '-all') {
  41       $all_tests = 1;
  42     }
  43 }
  44
  45 my $tgt_num = 0;
  46 my $imr = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  47 my $act = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  48 my $ti = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  49 my $n_srv = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  50 my $n_cli = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  51 my $bin_imr = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  52 my $bin_act = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
  53 my @a_cli;
  54 my @a_srv;
  55 my @p_srv;
  56 my $index;
  57 for ($index = 0; $index < $num_srvr; ++$index) {
  58   push(@a_cli, PerlACE::TestTarget::create_target (++$tgt_num)) || die "Create a_cli target $tgt_num failed\n";
  59   push(@a_srv, PerlACE::TestTarget::create_target (++$tgt_num)) || die "Create a_srv target $tgt_num failed\n";
  60   push(@p_srv, PerlACE::TestTarget::create_target (++$tgt_num)) || die "Create p_srv target $tgt_num failed\n";
  61 }
  62
  63 my $replica_imr;
  64 if ($replica) {
  65     $replica_imr = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create replica_imr target $tgt_num failed\n";
  66     $replica_imr->AddLibPath ("$ENV{ACE_ROOT}/lib");
  67 }
  68
  69 $imr->AddLibPath ("$ENV{ACE_ROOT}/lib");
  70
  71 my $imriorfile = "imr_locator.ior";
  72 my $actiorfile = "imr_activator.ior";
  73 my $primaryiorfile = "ImR_ReplicaPrimary.ior";
  74 my $backupiorfile = "ImR_ReplicaBackup.ior";
  75 my $replica_imriorfile = "replica_imr_locator.ior";
  76 my $nesteaiorfile = "nestea.ior";
  77 my $nestea_dat = "nestea.dat";
  78
  79 my $stdout_file         = "test.out";
  80 my $stderr_file         = "test.err";
  81
  82 my $n_cli_nesteaiorfile = $n_cli->LocalFile ($nesteaiorfile);
  83
  84 my $refstyle = " -ORBObjRefStyle URL";
  85 my $imr_refstyle = "";# = " -ORBObjRefStyle URL";
  86 my $protocol = "iiop";
  87 my $imr_host = $imr->HostName ();
  88 my $port = 12345;
  89 my $endpoint = "-ORBEndpoint " . "$protocol" . "://:" . $port;
  90
  91 my $IMR = $imr->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_locator");
  92 my $ACT = $act->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_activator");
  93 my $TI = $ti->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_imr");
  94
  95 # We want the tao_imr executable to be found exactly in the path
  96 # given, without being modified by the value of -ExeSubDir.
  97 # So, we tell its Process object to ignore the setting of -ExeSubDir.
  98
  99 $TI->IgnoreExeSubDir (1);
 100
 101 my $replica_IMR;
 102 if ($replica) {
 103     $replica_IMR = $replica_imr->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_locator");
 104 }
 105
 106 my @airplaneiorfile;
 107 my @a_cli_airplaneiorfile;
 108 for ($index = 0; $index < $num_srvr; ++$index) {
 109   push(@airplaneiorfile, "airplane$index.ior");
 110   push(@a_cli_airplaneiorfile, $a_cli[$index]->LocalFile ($airplaneiorfile[$index]));
 111 }
 112
 113 sub create_acli {
 114     my $select = shift;
 115     return $a_cli[$select]->CreateProcess ("airplane_client", " -k file://$a_cli_airplaneiorfile[$select] ");
 116 }
 117
 118 sub create_ncli {
 119     return $n_cli->CreateProcess ("nestea_client", " -k file://$n_cli_nesteaiorfile ");
 120 }
 121
 122 my @A_SRV;
 123 my @a_srv_name;
 124 my @A_SRV_cmd;
 125 my @imr_A_SRV_cmd;
 126 my @P_SRV;
 127 my @P_SRV_cmd;
 128 my @imr_P_SRV_cmd;
 129 for ($index = 0; $index < $num_srvr; ++$index) {
 130   push(@a_srv_name, "airplane_server$index");
 131   push(@A_SRV, $a_srv[$index]->CreateProcess ("airplane_server"));
 132   my $p_srv_name = "persist server$index";
 133   push(@P_SRV, $p_srv[$index]->CreateProcess ($p_srv_name));
 134
 135   push(@A_SRV_cmd, $A_SRV[$index]->Executable());
 136   push(@imr_A_SRV_cmd, $imr->LocalFile ($A_SRV_cmd[$index]));
 137   push(@P_SRV_cmd, $P_SRV[$index]->Executable());
 138   push(@imr_P_SRV_cmd, $imr->LocalFile ($P_SRV_cmd[$index]));
 139 }
 140
 141 my @A_CLI;
 142 for ($index = 0; $index < $num_srvr; ++$index) {
 143   push(@A_CLI, create_acli($index));
 144 }
 145 my $N_SRV = $a_cli[0]->CreateProcess ("nestea_server");
 146 my $N_CLI = create_ncli();
 147
 148 my $N_SRV_cmd = $N_SRV->Executable();
 149 my $imr_N_SRV_cmd = $imr->LocalFile ($N_SRV_cmd);
 150
 151
 152 ###############################################################################
 153 # Helper subroutines
 154 ###############################################################################
 155
 156 sub setup_repo
 157 {
 158     my $repo_ref = shift;
 159     my $the_imr = shift;
 160     my $the_IMR = shift;
 161     my $the_imriorfile = shift;
 162     my $the_act = shift;
 163     my $the_ACT = shift;
 164     my $the_actiorfile = shift;
 165     my $the_ti = shift;
 166     my $the_TI = shift;
 167     my $port = shift;
 168     my $replication_role = shift;
 169     my $the_replicaiorfile = shift;
 170     my $explicit_act = shift;
 171
 172     my $the_imr_imriorfile = $the_imr->LocalFile ($the_imriorfile);
 173     my $the_act_imriorfile = $the_act->LocalFile ($the_imriorfile);
 174     my $the_ti_imriorfile = $the_ti->LocalFile ($the_imriorfile);
 175     my $the_act_actiorfile = $the_act->LocalFile ($the_actiorfile);
 176     my $the_imr_replicaiorfile = $the_imr->LocalFile ($the_replicaiorfile);
 177
 178     $repo_ref->{imr} = $the_imr;
 179     $repo_ref->{IMR} = $the_IMR;
 180     $repo_ref->{imriorfile} = $the_imriorfile;
 181     $repo_ref->{imr_imriorfile} = $the_imr_imriorfile;
 182     $repo_ref->{imr_endpoint_flag} = "-ORBEndpoint iiop://:$port ";
 183     if ($replica) {
 184         $repo_ref->{imr_backing_store} = ".";
 185         $repo_ref->{imr_backing_store_flag} =
 186           "--directory $repo_ref->{imr_backing_store} $replication_role ";
 187     }
 188     $repo_ref->{replicaiorfile} = $the_replicaiorfile;
 189     $repo_ref->{imr_replicaiorfile} = $the_imr_replicaiorfile;
 190
 191     $repo_ref->{act} = $the_act;
 192     $repo_ref->{ACT} = $the_ACT;
 193     $repo_ref->{actiorfile} = $the_actiorfile;
 194     $repo_ref->{act_imriorfile} = $the_act_imriorfile;
 195     $repo_ref->{act_actiorfile} = $the_act_actiorfile;
 196     if (defined($explicit_act)) {
 197         $repo_ref->{server_act_flag} = "-l $explicit_act ";
 198         $repo_ref->{act_explicit_flag} = "-n $explicit_act ";
 199     }
 200
 201     $repo_ref->{ti} = $the_ti;
 202     $repo_ref->{TI} = $the_TI;
 203     $repo_ref->{ti_imriorfile} = $the_ti_imriorfile;
 204 }
 205
 206 ###############################################################################
 207
 208 sub cleanup_replication
 209 {
 210     my $dir = shift;
 211     if (!defined($dir)) {
 212         $dir = ".";
 213     }
 214
 215     my $listings = "$dir/imr_listing.xml";
 216     my $fnd = 0;
 217     if (open FILE, "<$listings") {
 218         while (<FILE>) {
 219             if ($_ =~ /fname="([^"]+)"?/) {
 220                 $fnd = 1;
 221                 my $file = "$dir/$1";
 222                 test_info("deleting $file\n");
 223                 $imr->DeleteFile ($file);
 224                 $imr->DeleteFile ($file . ".bak");
 225             }
 226         }
 227          close FILE;
 228     }
 229
 230 #   If the primary listings file has been corrupt then perform the
 231 #   deletions from the backup file.
 232
 233     if (!$fnd) {
 234        if (open FILE, "<$listings" . ".bak") {
 235            while (<FILE>) {
 236                if ($_ =~ /fname="([^"]+)"?/) {
 237                    my $file = "$dir/$1";
 238                    test_info("deleting $file\n");
 239                    $imr->DeleteFile ($file);
 240                    $imr->DeleteFile ($file . ".bak");
 241                }
 242            }
 243             close FILE;
 244        }
 245     }
 246     test_info("deleting $listings\n");
 247     $imr->DeleteFile ("$listings");
 248     $imr->DeleteFile ("$listings" . ".bak");
 249     $imr->DeleteFile ("$dir/$primaryiorfile");
 250     $imr->DeleteFile ("$dir/$backupiorfile");
 251 }
 252
 253 ###############################################################################
 254
 255 sub test_info {
 256     if ($test_debug_level < 10) {
 257         return;
 258     }
 259
 260     my $info = shift;
 261     print "$info";
 262 }
 263
 264 ###############################################################################
 265
 266 sub kill_imr
 267 {
 268     $IMR->Kill (); $IMR->TimedWait (1);
 269     if ($replica) {
 270       $replica_IMR->Kill (); $replica_IMR->TimedWait (1);
 271     }
 272 }
 273
 274 ###############################################################################
 275
 276 sub kill_act
 277 {
 278     $ACT->Kill (); $ACT->TimedWait (1);
 279 }
 280
 281 ###############################################################################
 282
 283 sub add_servers
 284 {
 285     my $repo_ref = shift;
 286     my $iorfiles_ref = shift;
 287
 288     for ($index = 0; $index < $num_srvr; ++$index) {
 289         print "\n\nadding server $index using tao_imr\n";
 290         $repo_ref->[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
 291             "=file://$repo_ref->[$index]->{ti_imriorfile} $refstyle " .
 292             "add $a_srv_name[$index] -c \"$imr_A_SRV_cmd[$index] " .
 293             "-o $iorfiles_ref->[$index] -s $a_srv_name[$index]\" " .
 294             "$repo_ref->[$index]->{server_act_flag} ");
 295         my $TI_status = $repo_ref->[$index]->{TI}->SpawnWaitKill (
 296             $repo_ref->[$index]->{ti}->ProcessStartWaitInterval());
 297         if ($TI_status != 0) {
 298             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 299             kill_act();
 300             kill_imr();
 301             return 1;
 302         }
 303         print "added server $index with the locator using tao_imr\n";
 304     }
 305     return 0;
 306 }
 307
 308 ###############################################################################
 309
 310 sub add_servers_again
 311 {
 312     my $repo_ref = shift;
 313     my $iorfiles_ref = shift;
 314
 315     for ($index = 0; $index < $num_srvr; ++$index) {
 316         print "\n\nadding server $index again using tao_imr\n";
 317         $repo_ref->[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
 318             "=file://$repo_ref->[$index]->{ti_imriorfile} $refstyle " .
 319             "add $a_srv_name[$index] -c \"$imr_A_SRV_cmd[$index] " .
 320             "-o $iorfiles_ref->[$index] -s $a_srv_name[$index]\" " .
 321             "$repo_ref->[$index]->{server_act_flag} ");
 322         my $TI_status = $repo_ref->[$index]->{TI}->SpawnWaitKill (
 323             $repo_ref->[$index]->{ti}->ProcessStartWaitInterval());
 324         if ($TI_status eq 0) {
 325             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 326             kill_act();
 327             kill_imr();
 328             return 1;
 329         }
 330         print "tao_imr returned $TI_status when attempting to add server $index again\n";
 331     }
 332     return 0;
 333 }
 334
 335 ###############################################################################
 336
 337 sub remove_servers
 338 {
 339     my $repo_ref = shift;
 340
 341     for ($index = 0; $index < $num_srvr; ++$index) {
 342         print "\n\nremoving server $index using tao_imr\n";
 343         $repo_ref->[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
 344             "=file://$repo_ref->[$index]->{ti_imriorfile} $refstyle " .
 345             "remove $a_srv_name[$index]");
 346         my $TI_status = $repo_ref->[$index]->{TI}->SpawnWaitKill (
 347             $repo_ref->[$index]->{ti}->ProcessStartWaitInterval());
 348         if ($TI_status != 0) {
 349             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 350             kill_act();
 351             kill_imr();
 352             return 1;
 353         }
 354         print "removed server $index\n";
 355     }
 356     return 0;
 357 }
 358
 359 ###############################################################################
 360
 361 sub start_clients
 362 {
 363     for ($index = 0; $index < $num_srvr; ++$index) {
 364         test_info("starting client for $a_srv_name[$index]=" .
 365           $A_CLI[$index]->CommandLine() . "\n");
 366         my $A_CLI_status = $A_CLI[$index]->Spawn ();
 367         if ($A_CLI_status != 0) {
 368             print STDERR
 369               "ERROR: Airplane Client ($index) failed to spawn returning $A_CLI_status\n";
 370             kill_then_timed_wait(\@A_SRV, 1);
 371             $ACT->Kill (); $ACT->TimedWait (1);
 372             $IMR->Kill (); $IMR->TimedWait (1);
 373             return 1;
 374         }
 375     }
 376     return 0;
 377 }
 378
 379 ###############################################################################
 380
 381 sub stop_clients
 382 {
 383     for ($index = 0; $index < $num_srvr; ++$index) {
 384         my $A_CLI_status = $A_CLI[$index]->WaitKill ($a_cli[$index]->ProcessStartWaitInterval());
 385         if ($A_CLI_status != 0) {
 386             print STDERR "ERROR: Airplane Client $index returned $A_CLI_status\n";
 387             return 1;
 388         }
 389         print "\n\nstopped client $index\n\n\n";
 390     }
 391     return 0;
 392 }
 393
 394 ###############################################################################
 395
 396 sub shutdown_servers_using_tao_imr
 397 {
 398     my $repo_ref = shift;
 399     for ($index = 0; $index < $num_srvr; ++$index) {
 400         print "\n\nshutting down server $index using tao_imr\n";
 401         $repo_ref->[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
 402             "=file://$repo_ref->[$index]->{ti_imriorfile} $refstyle " .
 403             "shutdown $a_srv_name[$index]");
 404         my $TI_status = $repo_ref->[$index]->{TI}->SpawnWaitKill (
 405             $repo_ref->[$index]->{ti}->ProcessStartWaitInterval());
 406         if ($TI_status != 0 ) {
 407             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 408             kill_act();
 409             kill_imr();
 410             return 1;
 411         }
 412         print "\n\nshut down server $index using tao_imr\n";
 413     }
 414     return 0;
 415 }
 416
 417 ###############################################################################
 418 # Use the TAO IMR through the provided IOR
 419
 420 sub shutdown_servers_using_ior
 421 {
 422     my $provided_ior = shift;
 423     my $target = shift;
 424
 425     for ($index = 0; $index < $num_srvr; ++$index) {
 426         print "\n\nshutting down server $index using tao_imr\n";
 427         $TI->Arguments ("-ORBInitRef ImplRepoService" .
 428             "=file://$provided_ior $refstyle " .
 429             "shutdown $a_srv_name[$index]");
 430         my $TI_status = $TI->SpawnWaitKill (
 431             $target->ProcessStartWaitInterval());
 432         if ($TI_status == 5) {
 433             print STDERR "INFO: tao_imr ($index) returned NOT_FOUND\n";
 434             $TI_status = 0
 435         }
 436         elsif ($TI_status != 0) {
 437             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 438             kill_act();
 439             kill_imr();
 440             return 1;
 441         }
 442         print "\n\nshut down server $index using tao_imr\n";
 443     }
 444     return 0;
 445 }
 446
 447 ###############################################################################
 448
 449 sub wait_for_imr
 450 {
 451     my $repo_ref = shift;
 452     my $filekey = shift;
 453
 454     if (!defined($filekey)) {
 455         $filekey = "imriorfile";
 456     }
 457     my $imr_only = ($filekey eq "imriorfile");
 458
 459     if ($repo_ref->{imr}->WaitForFileTimed (
 460                 $repo_ref->{$filekey},
 461                 $repo_ref->{imr}->ProcessStartWaitInterval()) == -1) {
 462         print STDERR "ERROR: cannot find file <" .
 463           $repo_ref->{"imr_$filekey"} . ">\n";
 464         kill_imr();
 465         return 1;
 466     }
 467     if ($repo_ref->{imr}->GetFile ($repo_ref->{$filekey}) == -1) {
 468         print STDERR "ERROR: cannot retrieve file <" .
 469           $repo_ref->{"imr_$filekey"} . ">\n";
 470         kill_imr();
 471         return 1;
 472     }
 473     if (!$imr_only) {
 474         if ($repo_ref->{act}->PutFile ($repo_ref->{$filekey}) == -1) {
 475             print STDERR "ERROR: cannot set file <" .
 476               $repo_ref->{"act_$filekey"} . ">\n";
 477             kill_imr();
 478             return 1;
 479         }
 480         if ($repo_ref->{ti}->PutFile ($repo_ref->{$filekey}) == -1) {
 481             print STDERR "ERROR: cannot set file <" .
 482               $repo_ref->{"act_$filekey"} . ">\n";
 483             kill_imr();
 484             return 1;
 485         }
 486     }
 487     return 0;
 488 }
 489
 490 ###############################################################################
 491
 492 sub kill_then_timed_wait
 493 {
 494   my $srvrs = shift;
 495   my $time = shift;
 496   my $length = scalar(@{$srvrs});
 497   for ($index = 0; $index < $length; ++$index) {
 498     $srvrs->[$index]->Kill (); $srvrs->[$index]->TimedWait (1);
 499   }
 500 }
 501
 502
 503 ###############################################################################
 504
 505 sub redirect_output
 506 {
 507
 508     my $test_stdout_file = shift;
 509     my $test_stderr_file = shift;
 510     open (OLDOUT, ">&", \*STDOUT) or die "Can't dup STDOUT: $!";
 511     open (OLDERR, ">&", \*STDERR) or die "Can't dup STDERR: $!";
 512     open STDOUT, '>', $test_stdout_file;
 513     open STDERR, '>', $test_stderr_file;
 514 }
 515
 516 ###############################################################################
 517
 518 sub restore_output()
 519 {
 520     open (STDERR, ">&OLDERR") or die "Can't dup OLDERR: $!";
 521     open (STDOUT, ">&OLDOUT") or die "Can't dup OLDOUT: $!";
 522 }
 523
 524
 525 ###############################################################################
 526 # The Tests
 527 ###############################################################################
 528
 529 sub airplane_test
 530 {
 531     my $status = 0;
 532
 533     my @a_srv_airplaneiorfile;
 534     for ($index = 0; $index < $num_srvr; ++$index) {
 535         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
 536         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
 537         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
 538
 539         $A_SRV[$index]->Arguments (
 540           "-o $a_srv_airplaneiorfile[$index] $refstyle -ORBDebugLevel " .
 541           "$srv_debug_level -s $a_srv_name[$index]");
 542         my $A_SRV_status = $A_SRV[$index]->Spawn ();
 543         if ($A_SRV_status != 0) {
 544             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
 545             return 1;
 546         }
 547     }
 548
 549     for ($index = 0; $index < $num_srvr; ++$index) {
 550         if ($a_srv[$index]->WaitForFileTimed ($airplaneiorfile[$index],$a_srv[$index]->ProcessStartWaitInterval()) == -1) {
 551             print STDERR "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
 552             kill_then_timed_wait(\@A_SRV, 1);
 553             return 1;
 554         }
 555         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
 556             print STDERR "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
 557             kill_then_timed_wait(\@A_SRV, 1);
 558             return 1;
 559         }
 560         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
 561             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
 562             kill_then_timed_wait(\@A_SRV, 1);
 563             return 1;
 564         }
 565     }
 566
 567     for ($index = 0; $index < $num_srvr; ++$index) {
 568         my $A_CLI_status = $A_CLI[$index]->SpawnWaitKill ($a_cli[$index]->ProcessStartWaitInterval());
 569         if ($A_CLI_status != 0) {
 570             print STDERR "ERROR: Airplane Client returned $A_CLI_status\n";
 571             $status = 1;
 572         }
 573     }
 574
 575     for ($index = 0; $index < $num_srvr; ++$index) {
 576         my $A_SRV_status = $A_SRV[$index]->TerminateWaitKill ($a_srv[$index]->ProcessStopWaitInterval());
 577         if ($A_SRV_status != 0) {
 578             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
 579             $status = 1;
 580         }
 581
 582         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
 583         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
 584     }
 585     return $status;
 586 }
 587
 588 ###############################################################################
 589
 590 sub nestea_test
 591 {
 592     my $status = 0;
 593
 594     my $n_srv_nesteaiorfile = $n_srv->LocalFile ($nesteaiorfile);
 595     $n_srv->DeleteFile ($nesteaiorfile);
 596     $n_cli->DeleteFile ($nesteaiorfile);
 597
 598     $N_SRV->Arguments ("-o $n_srv_nesteaiorfile $refstyle -ORBDebugLevel " .
 599                        "$srv_debug_level");
 600     my $N_SRV_status = $N_SRV->Spawn ();
 601     if ($N_SRV_status != 0) {
 602         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
 603         return 1;
 604     }
 605     if ($n_srv->WaitForFileTimed ($nesteaiorfile,$n_srv->ProcessStartWaitInterval()) == -1) {
 606         print STDERR "ERROR: cannot find file <$n_srv_nesteaiorfile>\n";
 607         $N_SRV->Kill (); $N_SRV->TimedWait (1);
 608         return 1;
 609     }
 610     if ($n_srv->GetFile ($nesteaiorfile) == -1) {
 611         print STDERR "ERROR: cannot retrieve file <$n_srv_nesteaiorfile>\n";
 612         $N_SRV->Kill (); $N_SRV->TimedWait (1);
 613         return 1;
 614     }
 615     if ($n_cli->PutFile ($nesteaiorfile) == -1) {
 616         print STDERR "ERROR: cannot set file <$n_cli_nesteaiorfile>\n";
 617         $N_SRV->Kill (); $N_SRV->TimedWait (1);
 618         return 1;
 619     }
 620
 621     my $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval());
 622     if ($N_CLI_status != 0) {
 623         print STDERR "ERROR: Nestea Client returned $N_CLI_status\n";
 624         $status = 1;
 625     }
 626
 627     $N_SRV_status = $N_SRV->TerminateWaitKill ($n_srv->ProcessStopWaitInterval());
 628     if ($N_SRV_status != 0) {
 629         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
 630         $status = 1;
 631     }
 632
 633     $n_srv->DeleteFile ($nesteaiorfile);
 634     $n_cli->DeleteFile ($nesteaiorfile);
 635
 636     return $status;
 637 }
 638
 639 ###############################################################################
 640
 641 sub manual_persistent_restart_test
 642 {
 643     my $backing_store_flag = shift;
 644     my $status = 0;
 645
 646     if ($srv_debug_level == 0) {
 647         $srv_debug_level = 1;
 648     }
 649
 650     test_info("manual_persistent_ir_test start\n");
 651
 652     my $backing_store;
 653     if ($backing_store_flag eq "-p") {
 654         $backing_store = "test.repo";
 655     } elsif ($backing_store_flag eq "--directory") {
 656         $backing_store = ".";
 657     } elsif ($backing_store_flag eq "-x") {
 658         $backing_store = "imr_backing_store.xml";
 659     }
 660
 661     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
 662     my $imr_storefile;
 663     if (defined($backing_store)) {
 664         $imr_storefile = $imr->LocalFile ($backing_store);
 665     }
 666     my $act_imriorfile = $act->LocalFile ($imriorfile);
 667     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
 668     my $act_actiorfile = $act->LocalFile ($actiorfile);
 669
 670     my @a_srv_imriorfile;
 671     my @imr_airplaneiorfile;
 672     my @a_srv_airplaneiorfile;
 673     for ($index = 0; $index < $num_srvr; ++$index) {
 674         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($imriorfile));
 675         push(@imr_airplaneiorfile, $imr->LocalFile ($airplaneiorfile[$index]));
 676         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
 677         $a_srv[$index]->DeleteFile ($imriorfile);
 678         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
 679         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
 680         $imr->DeleteFile ($airplaneiorfile[$index]);
 681     }
 682     $imr->DeleteFile ($imriorfile);
 683     if ($backing_store_flag eq "--directory") {
 684         cleanup_replication($imr_storefile);
 685     }
 686     $act->DeleteFile ($imriorfile);
 687     $ti->DeleteFile ($imriorfile);
 688     $act->DeleteFile ($actiorfile);
 689
 690     my $bin_imr_host = $bin_imr->HostName ();
 691     my $imr_initref = "-orbinitref ImplRepoService=corbaloc::$bin_imr_host:8888/ImplRepoService";# -ORBDebugLevel 10";
 692
 693     ## Be sure to start the ImR on a consistent endpoint, so that any created IORs
 694     ## remain valid even if the ImR restarts.
 695     my $imr_arguments = "-orbendpoint iiop://:8888 $backing_store_flag $imr_storefile -d $test_debug_level -o $imr_imriorfile $imr_refstyle";
 696     $IMR->Arguments ("$imr_arguments -e ");
 697     test_info("starting IMR=" . $IMR->CommandLine() . "\n");
 698     my $IMR_status = $IMR->Spawn ();
 699     if ($IMR_status != 0) {
 700         print STDERR "ERROR: ImR Service returned $IMR_status\n";
 701         return 1;
 702     }
 703     if ($imr->WaitForFileTimed ($imriorfile,
 704                                 $imr->ProcessStartWaitInterval()) == -1) {
 705         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
 706         $IMR->Kill (); $IMR->TimedWait (1);
 707         return 1;
 708     }
 709     if ($imr->GetFile ($imriorfile) == -1) {
 710         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
 711         $IMR->Kill (); $IMR->TimedWait (1);
 712         return 1;
 713     }
 714     if ($act->PutFile ($imriorfile) == -1) {
 715         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
 716         $IMR->Kill (); $IMR->TimedWait (1);
 717         return 1;
 718     }
 719     if ($ti->PutFile ($imriorfile) == -1) {
 720         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
 721         $IMR->Kill (); $IMR->TimedWait (1);
 722         return 1;
 723     }
 724
 725     for ($index = 0; $index < $num_srvr; ++$index) {
 726         if ($a_srv[$index]->PutFile ($imriorfile) == -1) {
 727             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
 728             $IMR->Kill (); $IMR->TimedWait (1);
 729             return 1;
 730         }
 731     }
 732
 733     $ACT->Arguments ("-d $test_debug_level -o $act_actiorfile -ORBInitRef ImplRepoService=file://$act_imriorfile");
 734     test_info("starting ACT=" . $ACT->CommandLine() . "\n");
 735     my $ACT_status = $ACT->Spawn ();
 736     if ($ACT_status != 0) {
 737         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
 738         return 1;
 739     }
 740
 741     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
 742         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
 743         $ACT->Kill (); $ACT->TimedWait (1);
 744         $IMR->Kill (); $IMR->TimedWait (1);
 745         return 1;
 746     }
 747     if ($act->GetFile ($actiorfile) == -1) {
 748         print STDERR "ERROR: cannot retrieve file <$act_actiorfile>\n";
 749         $ACT->Kill (); $ACT->TimedWait (1);
 750         $IMR->Kill (); $IMR->TimedWait (1);
 751         return 1;
 752     }
 753
 754     for ($index = 0; $index < $num_srvr; ++$index) {
 755         $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
 756         # Copy the server to a path with spaces to ensure that these
 757         # work corrrectly.
 758         copy ($A_SRV_cmd[$index], $P_SRV_cmd[$index]);
 759         chmod(0755, $P_SRV_cmd[$index]);
 760     }
 761
 762     for ($index = 0; $index < $num_srvr; ++$index) {
 763         # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
 764         $TI->Arguments (
 765           "-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
 766           "add $a_srv_name[$index] -a MANUAL -c \"\\\"$imr_P_SRV_cmd[$index]\\\" " .
 767           "$refstyle -s $a_srv_name[$index] -o $a_srv_airplaneiorfile[$index] -p $a_srv_name[$index].pid\"");
 768         test_info("starting TI=" . $TI->CommandLine() . "\n");
 769         my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 770         if ($TI_status != 0) {
 771             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
 772             $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
 773             $ACT->Kill (); $ACT->TimedWait (1);
 774             $IMR->Kill (); $IMR->TimedWait (1);
 775             return 1;
 776         }
 777     }
 778
 779     for ($index = 0; $index < $num_srvr; ++$index) {
 780       $TI->Arguments ("$imr_initref list $a_srv_name[$index]");
 781       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 782       if ($TI_status != 0) {
 783           print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 784           return 1;
 785       }
 786     }
 787
 788     for ($index = 0; $index < $num_srvr; ++$index) {
 789       $TI->Arguments ("$imr_initref start $a_srv_name[$index]");
 790       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 791       if ($TI_status != 0) {
 792           print STDERR "ERROR: tao_imr start -v returned $TI_status\n";
 793           return 1;
 794       }
 795     }
 796
 797     for ($index = 0; $index < $num_srvr; ++$index) {
 798       $TI->Arguments ("$imr_initref list $a_srv_name[$index]");
 799       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 800       if ($TI_status != 0) {
 801           print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 802           return 1;
 803       }
 804     }
 805
 806     for ($index = 0; $index < $num_srvr; ++$index) {
 807         test_info("killing server=" . $a_srv_name[$index] . "\n");
 808         my $server_pid = get_server_pid ($a_srv_name[$index]. ".pid");
 809         signal_server ($server_pid, "KILL");
 810     }
 811
 812     for ($index = 0; $index < $num_srvr; ++$index) {
 813       $TI->Arguments ("$imr_initref list $a_srv_name[$index]");
 814       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 815       if ($TI_status != 0) {
 816           print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 817           return 1;
 818       }
 819     }
 820
 821     for ($index = 0; $index < $num_srvr; ++$index) {
 822       $TI->Arguments ("$imr_initref start $a_srv_name[$index]");
 823       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 824       if ($TI_status != 0) {
 825           print STDERR "ERROR: tao_imr start -v returned $TI_status\n";
 826           return 1;
 827       }
 828     }
 829
 830     for ($index = 0; $index < $num_srvr; ++$index) {
 831       $TI->Arguments ("$imr_initref list $a_srv_name[$index]");
 832       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 833       if ($TI_status != 0) {
 834           print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 835           return 1;
 836       }
 837     }
 838
 839     test_info("killing IMR=" . $IMR->CommandLine() . "\n");
 840     my $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
 841     if ($IMR_status != 0) {
 842         print STDERR "ERROR: ImR returned $IMR_status\n";
 843         $ACT->Kill (); $ACT->TimedWait (1);
 844         return 1;
 845     }
 846
 847     test_info("killing ACT=" . $ACT->CommandLine() . "\n");
 848     $ACT->Kill (); $ACT->TimedWait (1);
 849
 850     for ($index = 0; $index < $num_srvr; ++$index) {
 851         test_info("killing server=" . $a_srv_name[$index] . "\n");
 852         my $server_pid = get_server_pid ($a_srv_name[$index]. ".pid");
 853         signal_server ($server_pid, "KILL");
 854     }
 855
 856     # Unlink so that we can wait on them again to know the server started.
 857     $imr->DeleteFile ($imriorfile);
 858     $act->DeleteFile ($imriorfile);
 859     $ti->DeleteFile ($imriorfile);
 860
 861     for ($index = 0; $index < $num_srvr; ++$index) {
 862         $a_srv[$index]->DeleteFile ($imriorfile);
 863     }
 864
 865     print "Restarting Implementation Repository.\n";
 866     $IMR->Arguments ("$imr_arguments");
 867     test_info("restarting IMR=" . $IMR->CommandLine() . "\n");
 868     $IMR_status = $IMR->Spawn ();
 869     if ($IMR_status != 0) {
 870         print STDERR "ERROR: ImR Service returned $IMR_status\n";
 871         $ACT->Kill (); $ACT->TimedWait (1);
 872         return 1;
 873     }
 874     if ($imr->WaitForFileTimed ($imriorfile,
 875                                 $imr->ProcessStartWaitInterval()) == -1) {
 876         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
 877         $ACT->Kill (); $ACT->TimedWait (1);
 878         $IMR->Kill (); $IMR->TimedWait (1);
 879         return 1;
 880     }
 881     if ($imr->GetFile ($imriorfile) == -1) {
 882         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
 883         $ACT->Kill (); $ACT->TimedWait (1);
 884         $IMR->Kill (); $IMR->TimedWait (1);
 885         return 1;
 886     }
 887     if ($act->PutFile ($imriorfile) == -1) {
 888         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
 889         $ACT->Kill (); $ACT->TimedWait (1);
 890         $IMR->Kill (); $IMR->TimedWait (1);
 891         return 1;
 892     }
 893     if ($ti->PutFile ($imriorfile) == -1) {
 894         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
 895         $ACT->Kill (); $ACT->TimedWait (1);
 896         $IMR->Kill (); $IMR->TimedWait (1);
 897         return 1;
 898     }
 899
 900     test_info("restarting ACT=" . $ACT->CommandLine() . "\n");
 901     my $ACT_status = $ACT->Spawn ();
 902     if ($ACT_status != 0) {
 903         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
 904         return 1;
 905     }
 906
 907     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
 908         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
 909         $ACT->Kill (); $ACT->TimedWait (1);
 910         $IMR->Kill (); $IMR->TimedWait (1);
 911         return 1;
 912     }
 913     if ($act->GetFile ($actiorfile) == -1) {
 914         print STDERR "ERROR: cannot retrieve file <$act_actiorfile>\n";
 915         $ACT->Kill (); $ACT->TimedWait (1);
 916         $IMR->Kill (); $IMR->TimedWait (1);
 917         return 1;
 918     }
 919
 920     for ($index = 0; $index < $num_srvr; ++$index) {
 921         if ($a_srv[$index]->PutFile ($imriorfile) == -1) {
 922             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
 923             $ACT->Kill (); $ACT->TimedWait (1);
 924             $IMR->Kill (); $IMR->TimedWait (1);
 925             return 1;
 926         }
 927     }
 928
 929     print "Listing TAO ImR Services after start of the activator\n";
 930
 931     $TI->Arguments ("$imr_initref list -v ");
 932     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 933     if ($TI_status != 0) {
 934         print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 935         return 1;
 936     }
 937
 938     print "Starting server again\n";
 939     for ($index = 0; $index < $num_srvr; ++$index) {
 940       $TI->Arguments ("$imr_initref start $a_srv_name[$index]");
 941       my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 942       if ($TI_status != 0) {
 943           print STDERR "ERROR: tao_imr start returned $TI_status\n";
 944           return 1;
 945       }
 946     }
 947
 948     $TI->Arguments ("$imr_initref list -v ");
 949     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
 950     if ($TI_status != 0) {
 951         print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
 952         return 1;
 953     }
 954
 955     my $A_CLI_status = $A_CLI[0]->SpawnWaitKill ($a_cli[0]->ProcessStartWaitInterval());
 956     if ($A_CLI_status != 0) {
 957         print STDERR "ERROR: airplane client returned $A_CLI_status\n";
 958         kill_then_timed_wait(\@A_SRV, 1);
 959         return 1;
 960     }
 961
 962     for ($index = 0; $index < $num_srvr; ++$index) {
 963         test_info("killing server=" . $a_srv_name[$index] . "\n");
 964         my $server_pid = get_server_pid ($a_srv_name[$index]. ".pid");
 965         signal_server ($server_pid, "KILL");
 966     }
 967
 968     for ($index = 0; $index < $num_srvr; ++$index) {
 969         test_info("killing ACT\n");
 970         $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
 971         if ($ACT_status != 0) {
 972             print STDERR "ERROR: Activator returned $ACT_status\n";
 973             $status = 1;
 974         }
 975     }
 976
 977     test_info("killing IMR\n");
 978     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
 979     if ($IMR_status != 0) {
 980         print STDERR "ERROR: ImR returned $IMR_status\n";
 981         $status = 1;
 982     }
 983
 984     if (!defined($backing_store)) {
 985         # startup ImR with "-e" flag to ensure cleanup of registry entries
 986         $IMR->Arguments ("$imr_arguments -e");
 987         test_info("starting IMR=" . $IMR->CommandLine() . "\n");
 988         $IMR_status = $IMR->Spawn ();
 989         if ($IMR_status != 0) {
 990             print STDERR "ERROR: ImR Service returned $IMR_status\n";
 991             return 1;
 992         }
 993
 994         $IMR->TimedWait ($imr->ProcessStartWaitInterval());
 995
 996         test_info("killing IMR\n");
 997         $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
 998         if ($IMR_status != 0) {
 999             print STDERR "ERROR: ImR returned $IMR_status\n";
1000             $status = 1;
1001         }
1002     }
1003     elsif ($backing_store_flag eq "--directory") {
1004         cleanup_replication($backing_store);
1005     }
1006     else {
1007         $imr->DeleteFile ($backing_store);
1008     }
1009     $imr->DeleteFile ($imriorfile);
1010     $act->DeleteFile ($imriorfile);
1011     $ti->DeleteFile ($imriorfile);
1012     $act->DeleteFile ($actiorfile);
1013     for ($index = 0; $index < $num_srvr; ++$index) {
1014         $imr->DeleteFile ($airplaneiorfile[$index]);
1015         $a_srv[$index]->DeleteFile ($imriorfile);
1016         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
1017         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
1018         $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
1019     }
1020
1021     return $status;
1022 }
1023
1024
1025 ###############################################################################
1026
1027 sub nt_service_test_i
1028 {
1029     my ($imr_initref, $BIN_ACT, $BIN_IMR) = @_;
1030
1031     my $a_srv_airplaneiorfile = $a_srv[0]->LocalFile ($airplaneiorfile[0]);
1032
1033     print "Installing TAO ImR Services\n";
1034     $BIN_ACT->Arguments ("-c install $imr_initref -d $test_debug_level -ORBDebugLevel $debug_level");
1035     $BIN_IMR->Arguments ("-c install -d $test_debug_level -orbendpoint iiop://:8888");
1036
1037     my $BIN_IMR_status = $BIN_IMR->SpawnWaitKill ($bin_imr->ProcessStartWaitInterval());
1038     if ($BIN_IMR_status != 0) {
1039         print STDERR "ERROR: ImR Service returned $BIN_IMR_status\n";
1040         return 1;
1041     }
1042
1043     my $BIN_ACT_status = $BIN_ACT->SpawnWaitKill ($bin_act->ProcessStartWaitInterval());
1044     if ($BIN_ACT_status != 0) {
1045         print STDERR "ERROR: ImR Activator returned $BIN_ACT_status\n";
1046         return 1;
1047     }
1048
1049     # Starting the activator will also start the locator
1050     print "Starting TAO Implementation Repository Services\n";
1051     # Starting the activator should start the ImR automatically
1052     #system("net start taoimr 2>&1");
1053     my $net_start_status = system("net start taoimractivator 2>&1");
1054     if ($net_start_status != 0) {
1055         print STDERR "ERROR: Starting ImR Activator service returned $net_start_status\n";
1056         return 1;
1057     }
1058
1059     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
1060     $TI->Arguments ("$imr_initref add $a_srv_name[0] -c \"$imr_A_SRV_cmd[0] -s $a_srv_name[0]\" ".
1061                     "-w \"$ENV{ACE_ROOT}/lib\"");
1062     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1063     if ($TI_status != 0) {
1064         print STDERR "ERROR: tao_imr add $a_srv_name[0] returned $TI_status\n";
1065         return 1;
1066     }
1067
1068     $TI->Arguments ("$imr_initref list -v");
1069     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1070     if ($TI_status != 0) {
1071         print STDERR "ERROR: tao_imr list -v returned $TI_status\n";
1072         return 1;
1073     }
1074
1075     $TI->Arguments ("$imr_initref ior $a_srv_name[0] -f $a_srv_airplaneiorfile");
1076     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1077     if ($TI_status != 0) {
1078         print STDERR "ERROR: tao_imr ior $a_srv_name[0] returned $TI_status\n";
1079         return 1;
1080     }
1081     if ($a_srv[0]->WaitForFileTimed ($airplaneiorfile[0],$a_srv[0]->ProcessStartWaitInterval()) == -1) {
1082         print STDERR "ERROR: cannot find file <$a_srv_airplaneiorfile>\n";
1083         kill_then_timed_wait(\@A_SRV, 1);
1084         return 1;
1085     }
1086     if ($a_srv[0]->GetFile ($airplaneiorfile[0]) == -1) {
1087         print STDERR "ERROR: cannot retrieve file <$a_srv_airplaneiorfile>\n";
1088         kill_then_timed_wait(\@A_SRV, 1);
1089         return 1;
1090     }
1091     if ($a_cli[0]->PutFile ($airplaneiorfile[0]) == -1) {
1092         print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
1093         kill_then_timed_wait(\@A_SRV, 1);
1094         return 1;
1095     }
1096
1097     my $A_CLI_status = $A_CLI[0]->SpawnWaitKill ($a_cli[0]->ProcessStartWaitInterval()+5);
1098     if ($A_CLI_status != 0) {
1099         print STDERR "ERROR: airplane client returned $A_CLI_status\n";
1100         kill_then_timed_wait(\@A_SRV, 1);
1101         return 1;
1102     }
1103
1104     $TI->Arguments ("$imr_initref shutdown $a_srv_name[0]");
1105     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval()+5);
1106     if ($TI_status != 0) {
1107         print STDERR "ERROR: tao_imr shutdown $a_srv_name[0] returned $TI_status\n";
1108         kill_then_timed_wait(\@A_SRV, 1);
1109         return 1;
1110     }
1111
1112     return 0;
1113 }
1114
1115 ###############################################################################
1116
1117 sub nt_service_test
1118 {
1119     my $result = 0;
1120
1121     my $a_srv_airplaneiorfile = $a_srv[0]->LocalFile ($airplaneiorfile[0]);
1122     $a_srv[0]->DeleteFile ($airplaneiorfile[0]);
1123     $a_cli[0]->DeleteFile ($airplaneiorfile[0]);
1124
1125     my $bin_imr_host = $bin_imr->HostName ();
1126
1127     # Just to show that it's possible, this test uses corbaloc instead of ior file.
1128     my $imr_initref = "-orbinitref ImplRepoService=corbaloc::$bin_imr_host:8888/ImplRepoService";
1129
1130     # To avoid having to ensure that the LocalSystem account has the correct path
1131     # we simply copy the imr executables to the same directory as the DLL's.
1132     my $BIN_IMR = $bin_imr->CreateProcess ("$ENV{ACE_ROOT}/lib/tao_imr_locator","");
1133     my $BIN_ACT = $bin_act->CreateProcess ("$ENV{ACE_ROOT}/lib/tao_imr_activator","");
1134     $BIN_IMR->IgnoreExeSubDir(1);
1135     $BIN_ACT->IgnoreExeSubDir(1);
1136
1137     print "Copying ImplRepo services to the same location as the dlls.\n";
1138     $bin_imr->DeleteFile ($BIN_IMR->Executable ());
1139     copy ($IMR->Executable (), $BIN_IMR->Executable ());
1140     $bin_act->DeleteFile ($BIN_ACT->Executable ());
1141     copy ($ACT->Executable (), $BIN_ACT->Executable ());
1142
1143     print "Stopping any existing TAO ImR Services\n";
1144     system("net stop taoimractivator > nul 2>&1");
1145     system("net stop taoimr > nul 2>&1");
1146
1147     print "Removing any existing TAO ImR Services\n";
1148     $BIN_ACT->Arguments ("-c remove");
1149     $BIN_IMR->Arguments ("-c remove");
1150     my $BIN_ACT_status = $BIN_ACT->SpawnWaitKill ($bin_act->ProcessStartWaitInterval());
1151     if ($BIN_ACT_status < 0) {
1152         print STDERR "ERROR: BIN Activator returned $BIN_ACT_status\n";
1153     }
1154     my $BIN_IMR_status = $BIN_IMR->SpawnWaitKill ($bin_imr->ProcessStartWaitInterval());
1155     if ($BIN_IMR_status < 0) {
1156         print STDERR "ERROR: BIN ImR Service returned $BIN_IMR_status\n";
1157     }
1158
1159     $result = nt_service_test_i ($imr_initref, $BIN_ACT, $BIN_IMR);
1160
1161     print "Stopping TAO Implementation Repository Service\n";
1162     system("net stop taoimractivator 2>&1");
1163     system("net stop taoimr 2>&1");
1164
1165     print "Removing TAO ImR Services\n";
1166     $BIN_ACT->Arguments ("-c remove");
1167     $BIN_ACT_status = $BIN_ACT->SpawnWaitKill ($bin_act->ProcessStartWaitInterval());
1168     if ($BIN_ACT_status != 0) {
1169         print STDERR "ERROR: BIN Activator returned $BIN_ACT_status\n";
1170     }
1171     $BIN_IMR->Arguments ("-c remove");
1172     $BIN_IMR_status = $BIN_IMR->SpawnWaitKill ($bin_imr->ProcessStartWaitInterval());
1173     if ($BIN_IMR_status != 0) {
1174         print STDERR "ERROR: BIN ImR Service returned $BIN_IMR_status\n";
1175     }
1176
1177     print "Removing tao_imr_locator copy.\n";
1178     $bin_imr->DeleteFile ($BIN_IMR->Executable ());
1179     $bin_act->DeleteFile ($BIN_ACT->Executable ());
1180
1181     $a_srv[0]->DeleteFile ($airplaneiorfile[0]);
1182     $a_cli[0]->DeleteFile ($airplaneiorfile[0]);
1183
1184     return $result;
1185 }
1186
1187 ###############################################################################
1188
1189 sub airplane_ir_test
1190 {
1191     my $status = 0;
1192
1193     if ($srv_debug_level < 2 && $replica) {
1194         $srv_debug_level = 2;#"2 -ORBLogFile server.log";
1195     }
1196
1197     my $imr_port = 10001 + $imr->RandomPort ();
1198     print "\n\nimr_port=$imr_port\n";
1199     my %repo;
1200     setup_repo(\%repo, $imr, $IMR, $imriorfile, $act, $ACT, $actiorfile, $ti,
1201       $TI, $imr_port, "--primary", $backupiorfile);
1202
1203     my %backup_repo;
1204     if ($replica) {
1205         setup_repo(\%backup_repo, $replica_imr, $replica_IMR,
1206           $replica_imriorfile, $act, $ACT, $actiorfile,
1207           $ti, $TI, $imr_port + 1, "--backup",
1208           $primaryiorfile);
1209     }
1210
1211     my @repo_for_srvr;
1212     my @a_srv_imriorfile;
1213     my @imr_airplaneiorfile;
1214     my @a_srv_airplaneiorfile;
1215     for ($index = 0; $index < $num_srvr; ++$index) {
1216         if ($index == ($num_srvr - 1) && $replica) {
1217             push(@repo_for_srvr, \%backup_repo);
1218         }
1219         else {
1220             push(@repo_for_srvr, \%repo);
1221         }
1222
1223         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($repo_for_srvr[$index]->{imriorfile}));
1224         push(@imr_airplaneiorfile, $repo_for_srvr[$index]->{imr}->LocalFile ($airplaneiorfile[$index]));
1225         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
1226         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
1227         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
1228         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
1229         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
1230     }
1231     $imr->DeleteFile ($imriorfile);
1232     $act->DeleteFile ($imriorfile);
1233     $ti->DeleteFile ($imriorfile);
1234     $act->DeleteFile ($actiorfile);
1235     if ($replica) {
1236         $replica_imr->DeleteFile ($replica_imriorfile);
1237         cleanup_replication($repo{imr_backing_store});
1238     }
1239
1240     print "\n\nstarting IMR\n";
1241     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
1242       "$imr_refstyle $repo{imr_backing_store_flag} $repo{imr_endpoint_flag}");
1243     my $IMR_status = $repo{IMR}->Spawn ();
1244     if ($IMR_status != 0) {
1245         print STDERR "ERROR: ImR Service returned $IMR_status\n";
1246         return 1;
1247     }
1248     if ($replica) {
1249         if (wait_for_imr(\%backup_repo, "replicaiorfile")) {
1250             return 1;
1251         }
1252         print "\n\nstarting backup IMR\n";
1253         $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
1254           "$backup_repo{imr_imriorfile} $imr_refstyle " .
1255           "$backup_repo{imr_backing_store_flag} " .
1256           "$backup_repo{imr_endpoint_flag}");
1257         my $replica_IMR_status = $backup_repo{IMR}->Spawn ();
1258         if ($replica_IMR_status != 0) {
1259             print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
1260             return 1;
1261         }
1262         if (wait_for_imr(\%repo, "replicaiorfile")) {
1263             return 1;
1264         }
1265         print "started backup IMR\n";
1266     }
1267     if (wait_for_imr(\%repo)) {
1268         return 1;
1269     }
1270     print "started IMR\n";
1271
1272     for ($index = 0; $index < $num_srvr; ++$index) {
1273         if ($a_srv[$index]->PutFile ($repo_for_srvr[$index]->{imriorfile}) == -1) {
1274             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
1275             kill_imr();
1276             return 1;
1277         }
1278     }
1279
1280     print "\n\nstarting ACT\n";
1281     $repo{ACT}->Arguments ("-d $test_debug_level -o $repo{act_actiorfile} " .
1282       "-ORBInitRef ImplRepoService=file://$repo{act_imriorfile} $refstyle " .
1283       $repo{act_explicit_flag});
1284     my $ACT_status = $repo{ACT}->Spawn ();
1285     if ($ACT_status != 0) {
1286         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
1287         return 1;
1288     }
1289
1290     if ($repo{act}->WaitForFileTimed (
1291                   $repo{actiorfile},
1292                   $repo{act}->ProcessStartWaitInterval()) == -1) {
1293         print STDERR "ERROR: cannot find file <" .
1294           $repo{act_actiorfile} . ">\n";
1295         kill_act();
1296         kill_imr();
1297         return 1;
1298     }
1299     print "started ACT\n";
1300
1301     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
1302     # Can use update to add servers.
1303     for ($index = 0; $index < $num_srvr; ++$index) {
1304         print "\n\nstarting TI $index\n";
1305         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
1306           "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
1307           "update $a_srv_name[$index] -c \"$imr_A_SRV_cmd[$index] " .
1308           "-o $imr_airplaneiorfile[$index] -s $a_srv_name[$index]\" " .
1309           "$repo_for_srvr[$index]->{server_act_flag} ");
1310         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
1311           $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
1312         if ($TI_status != 0) {
1313             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
1314             kill_act();
1315             kill_imr();
1316             return 1;
1317         }
1318         print "stopped TI $index\n";
1319
1320         print "\n\nstarting srv $index\n";
1321         $A_SRV[$index]->Arguments (
1322           "-ORBUseIMR 1 -o $a_srv_airplaneiorfile[$index] $refstyle ".
1323           "-ORBInitRef ImplRepoService=file://$a_srv_imriorfile[$index] ".
1324           "-ORBDebugLevel $srv_debug_level -s $a_srv_name[$index] ");
1325         my $A_SRV_status = $A_SRV[$index]->Spawn ();
1326         if ($A_SRV_status != 0) {
1327             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
1328             return 1;
1329         }
1330     }
1331
1332     for ($index = 0; $index < $num_srvr; ++$index) {
1333         if ($a_srv[$index]->WaitForFileTimed ($airplaneiorfile[$index],$a_srv[$index]->ProcessStartWaitInterval()) == -1) {
1334             print STDERR "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
1335             kill_then_timed_wait(\@A_SRV, 1);
1336             kill_act();
1337             kill_imr();
1338             return 1;
1339         }
1340         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
1341             print STDERR "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
1342             kill_then_timed_wait(\@A_SRV, 1);
1343             kill_act();
1344             kill_imr();
1345             return 1;
1346         }
1347         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
1348             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
1349             kill_then_timed_wait(\@A_SRV, 1);
1350             kill_act();
1351             kill_imr();
1352             return 1;
1353         }
1354         print "\n\nstarted srv $index\n\n";
1355     }
1356
1357     for ($index = 0; $index < $num_srvr; ++$index) {
1358         print "\n\nstarting client $index\n";
1359         my $A_CLI_status = $A_CLI[$index]->Spawn ();
1360         if ($A_CLI_status != 0) {
1361             print STDERR "ERROR: Airplane Client $index failed to spawn returning $A_CLI_status\n";
1362             $status = 1;
1363         }
1364     }
1365
1366     for ($index = 0; $index < $num_srvr; ++$index) {
1367         my $A_CLI_status = $A_CLI[$index]->WaitKill ($a_cli[$index]->ProcessStartWaitInterval());
1368         if ($A_CLI_status != 0) {
1369             print STDERR "ERROR: Airplane Client $index returned $A_CLI_status\n";
1370             $status = 1;
1371         }
1372         print "\n\nstopped client $index\n\n\n";
1373     }
1374
1375     for ($index = 0; $index < $num_srvr; ++$index) {
1376         print "\n\nstarting TI $index (2)\n";
1377         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef " .
1378           "ImplRepoService=file://$repo_for_srvr[$index]->{ti_imriorfile} " .
1379           "$refstyle shutdown $a_srv_name[$index]");
1380
1381         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
1382           $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
1383         if ($TI_status != 0) {
1384             print STDERR "ERROR: tao_imr 1 ($index) returned $TI_status\n";
1385             $status = 1;
1386         }
1387         print "stopped TI $index (2)\n";
1388     }
1389
1390     for ($index = 0; $index < $num_srvr; ++$index) {
1391         print "\n\nstarting client $index\n";
1392         # This client should force a new airplane_server to be started
1393         my $A_CLI_status = $A_CLI[$index]->SpawnWaitKill (
1394           $a_cli[$index]->ProcessStartWaitInterval());
1395         if ($A_CLI_status != 0) {
1396             print STDERR "ERROR: restarted Airplane Client $index returned " .
1397               "$A_CLI_status\n";
1398             $status = 1;
1399         }
1400         print "stopped client $index\n";
1401
1402         print "\n\nstarting client $index\n";
1403         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef " .
1404           "ImplRepoService=file://$repo_for_srvr[$index]->{ti_imriorfile} " .
1405           "$refstyle shutdown $a_srv_name[$index]");
1406
1407         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
1408           $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
1409         if ($TI_status != 0) {
1410             print STDERR "ERROR: tao_imr 2 ($index) returned $TI_status\n";
1411             $status = 1;
1412         }
1413         print "stopped client $index\n";
1414     }
1415
1416     for ($index = 0; $index < $num_srvr; ++$index) {
1417         print "\n\nkilling srv $index\n";
1418         my $A_SRV_status = $A_SRV[$index]->WaitKill ($a_srv[$index]->ProcessStopWaitInterval());
1419         if ($A_SRV_status != 0) {
1420             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
1421             $status = 1;
1422         }
1423         print "stopped srv $index\n";
1424     }
1425
1426     print "\n\nkilling ACT\n";
1427     $ACT_status = $repo{ACT}->TerminateWaitKill (
1428       $repo{act}->ProcessStopWaitInterval());
1429     if ($ACT_status != 0) {
1430         print STDERR "ERROR: Activator returned $ACT_status\n";
1431         $status = 1;
1432     }
1433     print "stopped ACT\n";
1434
1435     print "\n\nkilling IMR\n";
1436     $IMR_status = $repo{IMR}->TerminateWaitKill (
1437       $repo{imr}->ProcessStopWaitInterval());
1438     if ($IMR_status != 0) {
1439         print STDERR "ERROR: ImR returned $IMR_status\n";
1440         $status = 1;
1441     }
1442     print "stopped IMR\n";
1443
1444     if ($replica) {
1445         print "\n\nkilling backup IMR\n";
1446         $IMR_status = $backup_repo{IMR}->TerminateWaitKill (
1447           $backup_repo{imr}->ProcessStopWaitInterval());
1448         if ($IMR_status != 0) {
1449             print STDERR "ERROR: replica ImR returned $IMR_status\n";
1450             $status = 1;
1451         }
1452         print "stopped backup IMR\n";
1453     }
1454
1455     $repo{imr}->DeleteFile ($repo{imriorfile});
1456     $repo{act}->DeleteFile ($repo{imriorfile});
1457     $repo{ti}->DeleteFile ($repo{imriorfile});
1458     $repo{act}->DeleteFile ($repo{actiorfile});
1459
1460     if ($replica) {
1461         $backup_repo{imr}->DeleteFile (
1462           $backup_repo{imriorfile});
1463         $backup_repo{act}->DeleteFile (
1464           $backup_repo{imriorfile});
1465         $backup_repo{ti}->DeleteFile (
1466           $backup_repo{imriorfile});
1467         $backup_repo{act}->DeleteFile (
1468           $backup_repo{actiorfile});
1469         cleanup_replication($repo{imr_backing_store});
1470     }
1471
1472     for ($index = 0; $index < $num_srvr; ++$index) {
1473         $repo_for_srvr[$index]->{imr}->DeleteFile (
1474           $airplaneiorfile[$index]);
1475         $a_srv[$index]->DeleteFile (
1476           $repo_for_srvr[$index]->{imriorfile});
1477         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
1478         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
1479     }
1480     return $status;
1481 }
1482
1483 ###############################################################################
1484
1485 sub nestea_ir_test
1486 {
1487     my $status = 0;
1488
1489     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
1490     my $act_imriorfile = $act->LocalFile ($imriorfile);
1491     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
1492     my $n_srv_imriorfile = $n_srv->LocalFile ($imriorfile);
1493     my $act_actiorfile = $act->LocalFile ($actiorfile);
1494     my $imr_nesteaiorfile = $imr->LocalFile ($nesteaiorfile);
1495     my $n_srv_nesteaiorfile = $n_srv->LocalFile ($nesteaiorfile);
1496     $imr->DeleteFile ($imriorfile);
1497     $act->DeleteFile ($imriorfile);
1498     $ti->DeleteFile ($imriorfile);
1499     $n_srv->DeleteFile ($imriorfile);
1500     $act->DeleteFile ($actiorfile);
1501     $imr->DeleteFile ($nesteaiorfile);
1502     $n_srv->DeleteFile ($nesteaiorfile);
1503     $n_cli->DeleteFile ($nesteaiorfile);
1504
1505     $IMR->Arguments ("-d $test_debug_level -o $imr_imriorfile $imr_refstyle");
1506     my $IMR_status = $IMR->Spawn ();
1507     if ($IMR_status != 0) {
1508         print STDERR "ERROR: ImR Service returned $IMR_status\n";
1509         return 1;
1510     }
1511     if ($imr->WaitForFileTimed ($imriorfile,
1512                                 $imr->ProcessStartWaitInterval()) == -1) {
1513         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
1514         $IMR->Kill (); $IMR->TimedWait (1);
1515         return 1;
1516     }
1517     if ($imr->GetFile ($imriorfile) == -1) {
1518         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
1519         $IMR->Kill (); $IMR->TimedWait (1);
1520         return 1;
1521     }
1522     if ($act->PutFile ($imriorfile) == -1) {
1523         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
1524         $IMR->Kill (); $IMR->TimedWait (1);
1525         return 1;
1526     }
1527     if ($ti->PutFile ($imriorfile) == -1) {
1528         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
1529         $IMR->Kill (); $IMR->TimedWait (1);
1530         return 1;
1531     }
1532     if ($n_srv->PutFile ($imriorfile) == -1) {
1533         print STDERR "ERROR: cannot set file <$n_srv_imriorfile>\n";
1534         $IMR->Kill (); $IMR->TimedWait (1);
1535         return 1;
1536     }
1537
1538     $ACT->Arguments (
1539       "-d $test_debug_level -o $act_actiorfile -orbobjrefstyle URL ".
1540       "-ORBInitRef ImplRepoService=file://$act_imriorfile");
1541     my $ACT_status = $ACT->Spawn ();
1542     if ($ACT_status != 0) {
1543         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
1544         return 1;
1545     }
1546     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
1547         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
1548         $ACT->Kill (); $ACT->TimedWait (1);
1549         $IMR->Kill (); $IMR->TimedWait (1);
1550         return 1;
1551     }
1552
1553     $N_SRV->Arguments (
1554       "-ORBUseIMR 1 -o $n_srv_nesteaiorfile -orbobjrefstyle URL ".
1555       "-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
1556       "-ORBDebugLevel $srv_debug_level");
1557     my $N_SRV_status = $N_SRV->Spawn ();
1558     if ($N_SRV_status != 0) {
1559         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
1560         return 1;
1561     }
1562     if ($n_srv->WaitForFileTimed ($nesteaiorfile,$n_srv->ProcessStartWaitInterval()) == -1) {
1563         print STDERR "ERROR: cannot find file <$n_srv_nesteaiorfile>\n";
1564         $N_SRV->Kill (); $N_SRV->TimedWait (1);
1565         $ACT->Kill (); $ACT->TimedWait (1);
1566         $IMR->Kill (); $IMR->TimedWait (1);
1567         return 1;
1568     }
1569     if ($n_srv->GetFile ($nesteaiorfile) == -1) {
1570         print STDERR "ERROR: cannot retrieve file <$n_srv_nesteaiorfile>\n";
1571         $N_SRV->Kill (); $N_SRV->TimedWait (1);
1572         $ACT->Kill (); $ACT->TimedWait (1);
1573         $IMR->Kill (); $IMR->TimedWait (1);
1574         return 1;
1575     }
1576     if ($n_cli->PutFile ($nesteaiorfile) == -1) {
1577         print STDERR "ERROR: cannot set file <$n_cli_nesteaiorfile>\n";
1578         $N_SRV->Kill (); $N_SRV->TimedWait (1);
1579         $ACT->Kill (); $ACT->TimedWait (1);
1580         $IMR->Kill (); $IMR->TimedWait (1);
1581         return 1;
1582     }
1583
1584     my $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval());
1585     if ($N_CLI_status != 0) {
1586         print STDERR "ERROR: Nestea Client 1 returned $N_CLI_status\n";
1587         $N_SRV->Kill (); $N_SRV->TimedWait (1);
1588         $ACT->Kill (); $ACT->TimedWait (1);
1589         $IMR->Kill (); $IMR->TimedWait (1);
1590         return 1;
1591     }
1592
1593     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1594                     " shutdown nestea_server");
1595
1596     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1597     if ($TI_status != 0) {
1598         print STDERR "ERROR: tao_imr 1 returned $TI_status\n";
1599         $N_SRV->Kill (); $N_SRV->TimedWait (1);
1600         $ACT->Kill (); $ACT->TimedWait (1);
1601         $IMR->Kill (); $IMR->TimedWait (1);
1602         return 1;
1603     }
1604
1605     $N_SRV_status = $N_SRV->WaitKill ($n_srv->ProcessStopWaitInterval());
1606     if ($N_SRV_status != 0) {
1607         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
1608         $ACT->Kill (); $ACT->TimedWait (1);
1609         $IMR->Kill (); $IMR->TimedWait (1);
1610         return 1;
1611     }
1612
1613     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
1614     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1615                     "update nestea_server -l $imr_host ".
1616                     "-c \"$imr_N_SRV_cmd -o $imr_nesteaiorfile\"");
1617     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1618     if ($TI_status != 0) {
1619         print STDERR "ERROR: tao_imr returned $TI_status\n";
1620         $ACT->Kill (); $ACT->TimedWait (1);
1621         $IMR->Kill (); $IMR->TimedWait (1);
1622         return 1;
1623     }
1624
1625     # This should cause the activator to spawn another server.
1626     $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval()+5);
1627     if ($N_CLI_status != 0) {
1628         print STDERR "ERROR: Nestea Client 2 returned $N_CLI_status\n";
1629         $status = 1;
1630     }
1631
1632     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1633                     "shutdown nestea_server");
1634     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1635     if ($TI_status != 0) {
1636         print STDERR "ERROR: tao_imr 1 returned $TI_status\n";
1637         $status = 1;
1638     }
1639
1640     # This should destroy the POA, causing another to be created the next time
1641     # the server is spawned.
1642     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1643                          "remove nestea_server");
1644     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1645     if ($TI_status != 0) {
1646         print STDERR "ERROR: tao_imr 1 returned $TI_status\n";
1647         $status = 1;
1648     }
1649
1650     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
1651     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1652                     "add nestea_server -c \"$imr_N_SRV_cmd -o $imr_nesteaiorfile\"");
1653     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1654     if ($TI_status != 0) {
1655         print STDERR "ERROR: tao_imr 1 returned $TI_status\n";
1656         $status = 1;
1657     }
1658
1659     # This should cause the activator to spawn another server.
1660     $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval()+5);
1661     if ($N_CLI_status != 0) {
1662         print STDERR "ERROR: Nestea Client 2 returned $N_CLI_status\n";
1663         $status = 1;
1664     }
1665
1666     # This call should block until the server shuts down
1667     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1668                          "shutdown nestea_server");
1669     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1670     if ($TI_status != 0) {
1671         print STDERR "ERROR: tao_imr 1 returned $TI_status\n";
1672         $status = 1;
1673     }
1674
1675     $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
1676     if ($ACT_status != 0) {
1677         print STDERR "ERROR: Activator returned $ACT_status\n";
1678         $status = 1;
1679     }
1680
1681     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
1682     if ($IMR_status != 0) {
1683         print STDERR "ERROR: ImR returned $IMR_status\n";
1684         $status = 1;
1685     }
1686
1687     $imr->DeleteFile ($imriorfile);
1688     $act->DeleteFile ($imriorfile);
1689     $ti->DeleteFile ($imriorfile);
1690     $n_srv->DeleteFile ($imriorfile);
1691     $act->DeleteFile ($actiorfile);
1692     $imr->DeleteFile ($nesteaiorfile);
1693     $n_srv->DeleteFile ($nesteaiorfile);
1694     $n_cli->DeleteFile ($nesteaiorfile);
1695
1696     return $status;
1697 }
1698
1699 ###############################################################################
1700
1701 sub perclient
1702 {
1703     my $status = 0;
1704
1705     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
1706     my $act_imriorfile = $act->LocalFile ($imriorfile);
1707     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
1708     my $act_actiorfile = $act->LocalFile ($actiorfile);
1709     my $imr_nesteaiorfile = $imr->LocalFile ($nesteaiorfile);
1710     $imr->DeleteFile ($imriorfile);
1711     $act->DeleteFile ($imriorfile);
1712     $ti->DeleteFile ($imriorfile);
1713     $act->DeleteFile ($actiorfile);
1714     $imr->DeleteFile ($nesteaiorfile);
1715     $n_cli->DeleteFile ($nesteaiorfile);
1716
1717     my $imr_host = $imr->HostName ();
1718
1719     # specify an endpoint so that we can use corbaloc url for the client.
1720     $IMR->Arguments (
1721       "-d $test_debug_level -orbendpoint iiop://:8888 ".
1722       "-o $imr_imriorfile $imr_refstyle -ORBDebugLevel $debug_level");
1723     my $IMR_status = $IMR->Spawn ();
1724     if ($IMR_status != 0) {
1725         print STDERR "ERROR: ImR Service returned $IMR_status\n";
1726         return 1;
1727     }
1728     if ($imr->WaitForFileTimed ($imriorfile,
1729                                 $imr->ProcessStartWaitInterval()) == -1) {
1730         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
1731         $IMR->Kill (); $IMR->TimedWait (1);
1732         return 1;
1733     }
1734     if ($imr->GetFile ($imriorfile) == -1) {
1735         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
1736         $IMR->Kill (); $IMR->TimedWait (1);
1737         return 1;
1738     }
1739     if ($act->PutFile ($imriorfile) == -1) {
1740         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
1741         $IMR->Kill (); $IMR->TimedWait (1);
1742         return 1;
1743     }
1744     if ($ti->PutFile ($imriorfile) == -1) {
1745         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
1746         $IMR->Kill (); $IMR->TimedWait (1);
1747         return 1;
1748     }
1749
1750     $ACT->Arguments ("-d $test_debug_level -o $act_actiorfile ".
1751                      "-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$act_imriorfile");
1752     my $ACT_status = $ACT->Spawn ();
1753     if ($ACT_status != 0) {
1754         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
1755         return 1;
1756     }
1757     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
1758         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
1759         $ACT->Kill (); $ACT->TimedWait (1);
1760         $IMR->Kill (); $IMR->TimedWait (1);
1761         return 1;
1762     }
1763
1764     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
1765     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1766                     "add nestea_server -a PER_CLIENT ".
1767                     "-c \"$imr_N_SRV_cmd -o $imr_nesteaiorfile\"");
1768     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1769     if ($TI_status != 0) {
1770         print STDERR "ERROR: tao_imr returned $TI_status\n";
1771         $ACT->Kill (); $ACT->TimedWait (1);
1772         $IMR->Kill (); $IMR->TimedWait (1);
1773         return 1;
1774     }
1775
1776     $N_CLI->Arguments("-k corbaloc::$imr_host:8888/nestea_server");
1777
1778     # Running the client should start a server instance
1779     my $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval()+5);
1780     if ($N_CLI_status != 0) {
1781         print STDERR "ERROR: Nestea Client 1 returned $N_CLI_status\n";
1782         $ACT->Kill (); $ACT->TimedWait (1);
1783         $IMR->Kill (); $IMR->TimedWait (1);
1784         return 1;
1785     }
1786     if ($n_cli->WaitForFileTimed ($nesteaiorfile,$a_cli[0]->ProcessStartWaitInterval()) == -1) {
1787         print STDERR "ERROR: cannot find file <$n_cli_nesteaiorfile>\n";
1788         $ACT->Kill (); $ACT->TimedWait (1);
1789         $IMR->Kill (); $IMR->TimedWait (1);
1790         return 1;
1791     }
1792
1793     $a_cli[0]->DeleteFile ($nesteaiorfile);
1794
1795     $N_CLI->Arguments("-s -k corbaloc::$imr_host:8888/nestea_server");
1796
1797     # Running the client again should start another server instance
1798     $N_CLI_status = $N_CLI->SpawnWaitKill ($n_cli->ProcessStartWaitInterval()+5);
1799     if ($N_CLI_status != 0) {
1800         print STDERR "ERROR: Nestea Client 1 returned $N_CLI_status\n";
1801         $ACT->Kill (); $ACT->TimedWait (1);
1802         $IMR->Kill (); $IMR->TimedWait (1);
1803         return 1;
1804     }
1805     if ($n_cli->WaitForFileTimed ($nesteaiorfile,$a_cli[0]->ProcessStartWaitInterval()) == -1) {
1806         print STDERR "ERROR: cannot find file <$n_cli_nesteaiorfile>\n";
1807         $ACT->Kill (); $ACT->TimedWait (1);
1808         $IMR->Kill (); $IMR->TimedWait (1);
1809         return 1;
1810     }
1811
1812     # Note : We have to wait long enough for the first server to self-destruct
1813     # or it will print out an exception when it can't notify the imr of its shutdown.
1814     sleep 20;
1815
1816     $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
1817     if ($ACT_status != 0) {
1818         print STDERR "ERROR: Activator returned $ACT_status\n";
1819         $status = 1;
1820     }
1821
1822     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
1823     if ($IMR_status != 0) {
1824         print STDERR "ERROR: ImR returned $IMR_status\n";
1825         $status = 1;
1826     }
1827
1828     $imr->DeleteFile ($imriorfile);
1829     $act->DeleteFile ($imriorfile);
1830     $ti->DeleteFile ($imriorfile);
1831     $act->DeleteFile ($actiorfile);
1832     $imr->DeleteFile ($nesteaiorfile);
1833     $n_cli->DeleteFile ($nesteaiorfile);
1834
1835     return $status;
1836 }
1837 ###############################################################################
1838
1839 sub shutdown_repo
1840 {
1841     my $status = 0;
1842
1843     my $testrepo = "test.repo";
1844
1845     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
1846     my $act_imriorfile = $act->LocalFile ($imriorfile);
1847     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
1848     my $act_actiorfile = $act->LocalFile ($actiorfile);
1849     my $imr_testrepo = $imr->LocalFile ($testrepo);
1850     $imr->DeleteFile ($imriorfile);
1851     $act->DeleteFile ($imriorfile);
1852     $ti->DeleteFile ($imriorfile);
1853     $act->DeleteFile ($actiorfile);
1854     $imr->DeleteFile ($testrepo);
1855
1856     # Specify an endpoint so that we can restart on the same port.
1857     # Specify persistence so that we can test that shutdown-repo -a works after reconnect
1858     $IMR->Arguments ("-p $imr_testrepo -d 1 -orbendpoint iiop://:8888 -o $imr_imriorfile $imr_refstyle ".
1859                      "-ORBDebugLevel $debug_level");
1860     my $IMR_status = $IMR->Spawn ();
1861     if ($IMR_status != 0) {
1862         print STDERR "ERROR: ImR Service returned $IMR_status\n";
1863         return 1;
1864     }
1865     if ($imr->WaitForFileTimed ($imriorfile,
1866                                 $imr->ProcessStartWaitInterval()) == -1) {
1867         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
1868         $IMR->Kill (); $IMR->TimedWait (1);
1869         return 1;
1870     }
1871     if ($imr->GetFile ($imriorfile) == -1) {
1872         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
1873         $IMR->Kill (); $IMR->TimedWait (1);
1874         return 1;
1875     }
1876     if ($act->PutFile ($imriorfile) == -1) {
1877         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
1878         $IMR->Kill (); $IMR->TimedWait (1);
1879         return 1;
1880     }
1881     if ($ti->PutFile ($imriorfile) == -1) {
1882         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
1883         $IMR->Kill (); $IMR->TimedWait (1);
1884         return 1;
1885     }
1886
1887     $ACT->Arguments ("-d 1 -o $act_actiorfile ".
1888                      "-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$act_imriorfile");
1889     my $ACT_status = $ACT->Spawn ();
1890     if ($ACT_status != 0) {
1891         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
1892         return 1;
1893     }
1894     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
1895         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
1896         $ACT->Kill (); $ACT->TimedWait (1);
1897         $IMR->Kill (); $IMR->TimedWait (1);
1898         return 1;
1899     }
1900
1901     # Kill the ImR, but leave the activator running
1902     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1903                     "shutdown-repo");
1904     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1905     if ($TI_status != 0) {
1906         print STDERR "ERROR: tao_imr returned $TI_status\n";
1907         $ACT->Kill (); $ACT->TimedWait (1);
1908         $IMR->Kill (); $IMR->TimedWait (1);
1909         return 1;
1910     }
1911
1912     $IMR_status = $IMR->WaitKill ($imr->ProcessStopWaitInterval());
1913     if ($IMR_status != 0) {
1914         print STDERR "ERROR: ImR returned $IMR_status\n";
1915         return 1;
1916     }
1917
1918     $imr->DeleteFile ($imriorfile);
1919
1920     $IMR_status = $IMR->Spawn ();
1921     if ($IMR_status != 0) {
1922         print STDERR "ERROR: ImR Service returned $IMR_status\n";
1923         return 1;
1924     }
1925     if ($imr->WaitForFileTimed ($imriorfile,
1926                                 $imr->ProcessStartWaitInterval()) == -1) {
1927         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
1928         $IMR->Kill (); $IMR->TimedWait (1);
1929         return 1;
1930     }
1931     if ($imr->GetFile ($imriorfile) == -1) {
1932         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
1933         $IMR->Kill (); $IMR->TimedWait (1);
1934         return 1;
1935     }
1936     if ($ti->PutFile ($imriorfile) == -1) {
1937         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
1938         $IMR->Kill (); $IMR->TimedWait (1);
1939         return 1;
1940     }
1941
1942     $TI->Arguments ("-orbobjrefstyle URL -ORBInitRef ImplRepoService=file://$ti_imriorfile ".
1943                     "shutdown-repo -a");
1944     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
1945     if ($TI_status != 0) {
1946         print STDERR "ERROR: tao_imr returned $TI_status\n";
1947         $status = 1;
1948     }
1949
1950     $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
1951     if ($ACT_status != 0) {
1952         print STDERR "ERROR: Activator returned $ACT_status\n";
1953         $status = 1;
1954     }
1955
1956     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
1957     if ($IMR_status != 0) {
1958         print STDERR "ERROR: ImR returned $IMR_status\n";
1959         $status = 1;
1960     }
1961
1962     $imr->DeleteFile ($imriorfile);
1963     $act->DeleteFile ($imriorfile);
1964     $ti->DeleteFile ($imriorfile);
1965     $act->DeleteFile ($actiorfile);
1966     $imr->DeleteFile ($testrepo);
1967
1968     return $status;
1969 }
1970
1971 ###############################################################################
1972
1973 sub persistent_ir_test
1974 {
1975     my $backing_store_flag = shift;
1976     my $status = 0;
1977
1978     if ($srv_debug_level == 0) {
1979         $srv_debug_level = 1;
1980     }
1981
1982     test_info("persistent_ir_test start\n");
1983
1984     my $backing_store;
1985     if ($backing_store_flag eq "-p") {
1986         $backing_store = "test.repo";
1987     } elsif ($backing_store_flag eq "--directory") {
1988         $backing_store = ".";
1989     } elsif ($backing_store_flag eq "-x") {
1990         $backing_store = "imr_backing_store.xml";
1991     }
1992
1993     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
1994     my $imr_storefile;
1995     if (defined($backing_store)) {
1996         $imr_storefile = $imr->LocalFile ($backing_store);
1997     }
1998     my $act_imriorfile = $act->LocalFile ($imriorfile);
1999     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
2000     my $act_actiorfile = $act->LocalFile ($actiorfile);
2001
2002     my @a_srv_imriorfile;
2003     my @imr_airplaneiorfile;
2004     my @a_srv_airplaneiorfile;
2005     for ($index = 0; $index < $num_srvr; ++$index) {
2006         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($imriorfile));
2007         push(@imr_airplaneiorfile, $imr->LocalFile ($airplaneiorfile[$index]));
2008         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
2009         $a_srv[$index]->DeleteFile ($imriorfile);
2010         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2011         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2012         $imr->DeleteFile ($airplaneiorfile[$index]);
2013     }
2014     $imr->DeleteFile ($imriorfile);
2015     if ($backing_store_flag eq "--directory") {
2016         cleanup_replication($imr_storefile);
2017     }
2018     $act->DeleteFile ($imriorfile);
2019     $ti->DeleteFile ($imriorfile);
2020     $act->DeleteFile ($actiorfile);
2021
2022     ## Be sure to start the ImR on a consistent endpoint, so that any created IORs
2023     ## remain valid even if the ImR restarts.
2024     my $imr_arguments = "-orbendpoint iiop://:8888 $backing_store_flag $imr_storefile -d $test_debug_level -o $imr_imriorfile $imr_refstyle ";
2025     $IMR->Arguments ("$imr_arguments -e ");
2026     test_info("starting IMR=" . $IMR->CommandLine() . "\n");
2027     my $IMR_status = $IMR->Spawn ();
2028     if ($IMR_status != 0) {
2029         print STDERR "ERROR: ImR Service returned $IMR_status\n";
2030         return 1;
2031     }
2032     if ($imr->WaitForFileTimed ($imriorfile,
2033                                 $imr->ProcessStartWaitInterval()) == -1) {
2034         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
2035         $IMR->Kill (); $IMR->TimedWait (1);
2036         return 1;
2037     }
2038     if ($imr->GetFile ($imriorfile) == -1) {
2039         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
2040         $IMR->Kill (); $IMR->TimedWait (1);
2041         return 1;
2042     }
2043     if ($act->PutFile ($imriorfile) == -1) {
2044         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
2045         $IMR->Kill (); $IMR->TimedWait (1);
2046         return 1;
2047     }
2048     if ($ti->PutFile ($imriorfile) == -1) {
2049         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
2050         $IMR->Kill (); $IMR->TimedWait (1);
2051         return 1;
2052     }
2053
2054     for ($index = 0; $index < $num_srvr; ++$index) {
2055         if ($a_srv[$index]->PutFile ($imriorfile) == -1) {
2056             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
2057             $IMR->Kill (); $IMR->TimedWait (1);
2058             return 1;
2059         }
2060     }
2061
2062     $ACT->Arguments ("-d $test_debug_level -o $act_actiorfile -ORBInitRef ImplRepoService=file://$act_imriorfile");
2063     test_info("starting ACT=" . $ACT->CommandLine() . "\n");
2064     my $ACT_status = $ACT->Spawn ();
2065     if ($ACT_status != 0) {
2066         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
2067         return 1;
2068     }
2069
2070     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
2071         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
2072         $ACT->Kill (); $ACT->TimedWait (1);
2073         $IMR->Kill (); $IMR->TimedWait (1);
2074         return 1;
2075     }
2076     if ($act->GetFile ($actiorfile) == -1) {
2077         print STDERR "ERROR: cannot retrieve file <$act_actiorfile>\n";
2078         $ACT->Kill (); $ACT->TimedWait (1);
2079         $IMR->Kill (); $IMR->TimedWait (1);
2080         return 1;
2081     }
2082
2083     for ($index = 0; $index < $num_srvr; ++$index) {
2084         $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
2085         # Copy the server to a path with spaces to ensure that these
2086         # work corrrectly.
2087         copy ($A_SRV_cmd[$index], $P_SRV_cmd[$index]);
2088         chmod(0755, $P_SRV_cmd[$index]);
2089     }
2090
2091     for ($index = 0; $index < $num_srvr; ++$index) {
2092         # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
2093         $TI->Arguments (
2094           "-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
2095           "add $a_srv_name[$index] -c \"\\\"$imr_P_SRV_cmd[$index]\\\" " .
2096           "$refstyle -s $a_srv_name[$index]\"");
2097         test_info("starting TI=" . $TI->CommandLine() . "\n");
2098         my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
2099         if ($TI_status != 0) {
2100             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
2101             $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
2102             $ACT->Kill (); $ACT->TimedWait (1);
2103             $IMR->Kill (); $IMR->TimedWait (1);
2104             return 1;
2105         }
2106     }
2107
2108     for ($index = 0; $index < $num_srvr; ++$index) {
2109         ## This will write out the imr-ified IOR. Note : If you don't use -orbendpoint
2110         ## when starting the ImR, then this IOR will no longer be valid when the ImR
2111         ## restarts below. You can fix this by creating a new valid IOR, or starting
2112         ## the ImR on a consistent endpoint.
2113         $A_SRV[$index]->Arguments (
2114           "-o $a_srv_airplaneiorfile[$index] -ORBUseIMR 1 $refstyle ".
2115           "-ORBInitRef ImplRepoService=file://$a_srv_imriorfile[$index] ".
2116           "-ORBDebugLevel $srv_debug_level -s $a_srv_name[$index]");
2117         test_info("starting $a_srv_name[$index]=" .
2118           $A_SRV[$index]->CommandLine() . "\n");
2119         my $A_SRV_status = $A_SRV[$index]->Spawn ();
2120         if ($A_SRV_status != 0) {
2121             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
2122             return 1;
2123         }
2124     }
2125
2126     for ($index = 0; $index < $num_srvr; ++$index) {
2127         if ($a_srv[$index]->WaitForFileTimed (
2128                   $airplaneiorfile[$index],
2129                   $a_srv[$index]->ProcessStartWaitInterval()) == -1) {
2130             print STDERR
2131               "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
2132             kill_then_timed_wait(\@A_SRV, 1);
2133             $ACT->Kill (); $ACT->TimedWait (1);
2134             $IMR->Kill (); $IMR->TimedWait (1);
2135             return 1;
2136         }
2137         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
2138             print STDERR
2139               "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
2140             kill_then_timed_wait(\@A_SRV, 1);
2141             $ACT->Kill (); $ACT->TimedWait (1);
2142             $IMR->Kill (); $IMR->TimedWait (1);
2143             return 1;
2144         }
2145         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
2146             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
2147             kill_then_timed_wait(\@A_SRV, 1);
2148             $ACT->Kill (); $ACT->TimedWait (1);
2149             $IMR->Kill (); $IMR->TimedWait (1);
2150             return 1;
2151         }
2152     }
2153
2154     for ($index = 0; $index < $num_srvr; ++$index) {
2155         test_info("starting client for $a_srv_name[$index]=" .
2156           $A_CLI[$index]->CommandLine() . "\n");
2157         my $A_CLI_status = $A_CLI[$index]->Spawn ();
2158         if ($A_CLI_status != 0) {
2159             print STDERR
2160               "ERROR: Airplane Client ($index) failed to spawn returning $A_CLI_status\n";
2161             kill_then_timed_wait(\@A_SRV, 1);
2162             $ACT->Kill (); $ACT->TimedWait (1);
2163             $IMR->Kill (); $IMR->TimedWait (1);
2164             return 1;
2165         }
2166     }
2167
2168     for ($index = 0; $index < $num_srvr; ++$index) {
2169         test_info("killing client for $a_srv_name[$index]\n");
2170         my $A_CLI_status = $A_CLI[$index]->WaitKill ($a_cli[$index]->ProcessStartWaitInterval());
2171         if ($A_CLI_status != 0) {
2172             print STDERR "ERROR: Airplane Client ($index) returned $A_CLI_status\n";
2173             kill_then_timed_wait(\@A_SRV, 1);
2174             $ACT->Kill (); $ACT->TimedWait (1);
2175             $IMR->Kill (); $IMR->TimedWait (1);
2176             return 1;
2177         }
2178     }
2179
2180     for ($index = 0; $index < $num_srvr; ++$index) {
2181         $TI->Arguments ("-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
2182                         "shutdown $a_srv_name[$index]");
2183         test_info("starting TI=" . $TI->CommandLine() . "\n");
2184         my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
2185         if ($TI_status != 0) {
2186             print STDERR "ERROR: tao_imr shutdown returned $TI_status\n";
2187             kill_then_timed_wait(\@A_SRV, 1);
2188             $ACT->Kill (); $ACT->TimedWait (1);
2189             $IMR->Kill (); $IMR->TimedWait (1);
2190             return 1;
2191         }
2192     }
2193
2194     for ($index = 0; $index < $num_srvr; ++$index) {
2195         test_info("killing $a_srv_name[$index]\n");
2196         my $A_SRV_status = $A_SRV[$index]->WaitKill ($a_srv[$index]->ProcessStartWaitInterval());
2197         if ($A_SRV_status != 0) {
2198             print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
2199             $ACT->Kill (); $ACT->TimedWait (1);
2200             $IMR->Kill (); $IMR->TimedWait (1);
2201             return 1;
2202         }
2203     }
2204
2205     for ($index = 0; $index < $num_srvr; ++$index) {
2206         test_info("starting client for $a_srv_name[$index]\n");
2207         # Should cause the activator to spawn another server.
2208         my $A_CLI_status = $A_CLI[$index]->Spawn ();
2209         if ($A_CLI_status != 0) {
2210             print STDERR "ERROR: Airplane Client 2 ($index) failed to spawn returning $A_CLI_status\n";
2211             $ACT->Kill (); $ACT->TimedWait (1);
2212             $IMR->Kill (); $IMR->TimedWait (1);
2213             return 1;
2214         }
2215     }
2216
2217     for ($index = 0; $index < $num_srvr; ++$index) {
2218         test_info("killing client for $a_srv_name[$index]\n");
2219         # Should cause the activator to spawn another server.
2220         my $A_CLI_status = $A_CLI[$index]->WaitKill ($a_cli[$index]->ProcessStartWaitInterval());
2221         if ($A_CLI_status != 0) {
2222             print STDERR "ERROR: Airplane Client 2 ($index) returned $A_CLI_status\n";
2223             $ACT->Kill (); $ACT->TimedWait (1);
2224             $IMR->Kill (); $IMR->TimedWait (1);
2225             return 1;
2226         }
2227     }
2228
2229     for ($index = 0; $index < $num_srvr; ++$index) {
2230         # Shutdown airplane_server
2231         $TI->Arguments ("-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
2232                         "shutdown $a_srv_name[$index]");
2233         test_info("starting TI=" . $TI->CommandLine() . "\n");
2234         my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
2235         if ($TI_status != 0) {
2236             print STDERR "ERROR: tao_imr shutdown 2 ($index) returned $TI_status\n";
2237             $ACT->Kill (); $ACT->TimedWait (1);
2238             $IMR->Kill (); $IMR->TimedWait (1);
2239             return 1;
2240         }
2241     }
2242
2243     for ($index = 0; $index < $num_srvr; ++$index) {
2244         test_info("killing IMR=" . $IMR->CommandLine() . "\n");
2245         my $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
2246         if ($IMR_status != 0) {
2247             print STDERR "ERROR: ImR returned $IMR_status\n";
2248             $ACT->Kill (); $ACT->TimedWait (1);
2249             return 1;
2250         }
2251     }
2252
2253     # Unlink so that we can wait on them again to know the server started.
2254     $imr->DeleteFile ($imriorfile);
2255     $act->DeleteFile ($imriorfile);
2256     $ti->DeleteFile ($imriorfile);
2257
2258     for ($index = 0; $index < $num_srvr; ++$index) {
2259         $a_srv[$index]->DeleteFile ($imriorfile);
2260     }
2261
2262     print "Restarting Implementation Repository.\n";
2263     $IMR->Arguments ("$imr_arguments ");
2264     test_info("restarting IMR=" . $IMR->CommandLine() . "\n");
2265     $IMR_status = $IMR->Spawn ();
2266     if ($IMR_status != 0) {
2267         print STDERR "ERROR: ImR Service returned $IMR_status\n";
2268         $ACT->Kill (); $ACT->TimedWait (1);
2269         return 1;
2270     }
2271     if ($imr->WaitForFileTimed ($imriorfile,
2272                                 $imr->ProcessStartWaitInterval()) == -1) {
2273         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
2274         $ACT->Kill (); $ACT->TimedWait (1);
2275         $IMR->Kill (); $IMR->TimedWait (1);
2276         return 1;
2277     }
2278     if ($imr->GetFile ($imriorfile) == -1) {
2279         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
2280         $ACT->Kill (); $ACT->TimedWait (1);
2281         $IMR->Kill (); $IMR->TimedWait (1);
2282         return 1;
2283     }
2284     if ($act->PutFile ($imriorfile) == -1) {
2285         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
2286         $ACT->Kill (); $ACT->TimedWait (1);
2287         $IMR->Kill (); $IMR->TimedWait (1);
2288         return 1;
2289     }
2290     if ($ti->PutFile ($imriorfile) == -1) {
2291         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
2292         $ACT->Kill (); $ACT->TimedWait (1);
2293         $IMR->Kill (); $IMR->TimedWait (1);
2294         return 1;
2295     }
2296
2297     for ($index = 0; $index < $num_srvr; ++$index) {
2298         if ($a_srv[$index]->PutFile ($imriorfile) == -1) {
2299             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
2300             $ACT->Kill (); $ACT->TimedWait (1);
2301             $IMR->Kill (); $IMR->TimedWait (1);
2302             return 1;
2303         }
2304     }
2305
2306     for ($index = 0; $index < $num_srvr; ++$index) {
2307         test_info("starting client for $a_srv_name[$index]\n");
2308         # Should cause the activator to spawn another server.
2309         my $A_CLI_status = $A_CLI[$index]->Spawn ();
2310         if ($A_CLI_status != 0) {
2311             print STDERR "ERROR: Airplane Client 3 ($index) failed to spawn returning $A_CLI_status\n";
2312             $status = 1;
2313         }
2314     }
2315
2316     for ($index = 0; $index < $num_srvr; ++$index) {
2317         test_info("killing client for $a_srv_name[$index]\n");
2318         # Should cause the activator to spawn another server.
2319         my $A_CLI_status = $A_CLI[$index]->WaitKill ($a_cli[$index]->ProcessStartWaitInterval()+5);
2320         if ($A_CLI_status != 0) {
2321             print STDERR "ERROR: Airplane Client 3 ($index) returned $A_CLI_status\n";
2322             $status = 1;
2323         }
2324     }
2325
2326     for ($index = 0; $index < $num_srvr; ++$index) {
2327         # Shutdown airplane_server
2328         $TI->Arguments ("-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
2329                         "shutdown $a_srv_name[$index]");
2330         test_info("starting TI=" . $TI->CommandLine() . "\n");
2331         my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
2332         if ($TI_status != 0) {
2333             print STDERR "ERROR: tao_imr shutdown 3 returned $TI_status\n";
2334             $status = 1;
2335         }
2336     }
2337
2338     for ($index = 0; $index < $num_srvr; ++$index) {
2339         test_info("killing ACT\n");
2340         $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
2341         if ($ACT_status != 0) {
2342             print STDERR "ERROR: Activator returned $ACT_status\n";
2343             $status = 1;
2344         }
2345     }
2346
2347     test_info("killing IMR\n");
2348     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
2349     if ($IMR_status != 0) {
2350         print STDERR "ERROR: ImR returned $IMR_status\n";
2351         $status = 1;
2352     }
2353
2354     if (!defined($backing_store)) {
2355         # startup ImR with "-e" flag to ensure cleanup of registry entries
2356         $IMR->Arguments ("$imr_arguments -e");
2357         test_info("starting IMR=" . $IMR->CommandLine() . "\n");
2358         $IMR_status = $IMR->Spawn ();
2359         if ($IMR_status != 0) {
2360             print STDERR "ERROR: ImR Service returned $IMR_status\n";
2361             return 1;
2362         }
2363
2364         $IMR->TimedWait ($imr->ProcessStartWaitInterval());
2365
2366         test_info("killing IMR\n");
2367         $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
2368         if ($IMR_status != 0) {
2369             print STDERR "ERROR: ImR returned $IMR_status\n";
2370             $status = 1;
2371         }
2372     }
2373     elsif ($backing_store_flag eq "--directory") {
2374         cleanup_replication($backing_store);
2375     }
2376     else {
2377         $imr->DeleteFile ($backing_store);
2378     }
2379     $imr->DeleteFile ($imriorfile);
2380     $act->DeleteFile ($imriorfile);
2381     $ti->DeleteFile ($imriorfile);
2382     $act->DeleteFile ($actiorfile);
2383     for ($index = 0; $index < $num_srvr; ++$index) {
2384         $imr->DeleteFile ($airplaneiorfile[$index]);
2385         $a_srv[$index]->DeleteFile ($imriorfile);
2386         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2387         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2388         $p_srv[$index]->DeleteFile ($P_SRV_cmd[$index]);
2389     }
2390
2391     return $status;
2392 }
2393
2394 ###############################################################################
2395
2396 sub failover_test
2397 {
2398     if (!$replica) {
2399         print STDERR "The failover test needs the -replica flag\n";
2400         return 0;
2401     }
2402
2403     my $status = 0;
2404
2405     my $imr_port = 10001 + $imr->RandomPort ();
2406     my $replica_imr_port = $imr_port + 10;
2407
2408     my %repo;
2409     setup_repo(\%repo, $imr, $IMR, $imriorfile, $act, $ACT, $actiorfile, $ti,
2410       $TI, $imr_port, "--primary", $backupiorfile);
2411
2412     my %backup_repo;
2413     setup_repo(\%backup_repo, $replica_imr, $replica_IMR, $replica_imriorfile,
2414       $act, $ACT, $actiorfile, $ti, $TI,
2415       $replica_imr_port, "--backup", $primaryiorfile);
2416
2417     my @repo_for_srvr;
2418     my @a_srv_imriorfile;
2419     my @imr_airplaneiorfile;
2420     my @a_srv_airplaneiorfile;
2421     for ($index = 0; $index < $num_srvr; ++$index) {
2422         if ($index == ($num_srvr - 1)) {
2423             push(@repo_for_srvr, \%backup_repo);
2424         } else {
2425             push(@repo_for_srvr, \%repo);
2426         }
2427
2428         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($repo_for_srvr[$index]->{imriorfile}));
2429         push(@imr_airplaneiorfile, $repo_for_srvr[$index]->{imr}->LocalFile ($airplaneiorfile[$index]));
2430         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
2431         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
2432         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2433         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2434         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
2435     }
2436     $imr->DeleteFile ($imriorfile);
2437     $imr->DeleteFile ($primaryiorfile);
2438     $act->DeleteFile ($imriorfile);
2439     $ti->DeleteFile ($imriorfile);
2440     $act->DeleteFile ($actiorfile);
2441     $replica_imr->DeleteFile ($replica_imriorfile);
2442     $replica_imr->DeleteFile ($backupiorfile);
2443
2444     cleanup_replication ();
2445
2446     print "\n\nstarting primary tao_imr_locator\n";
2447     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
2448         "$repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} ");
2449     my $IMR_status = $repo{IMR}->Spawn ();
2450     if ($IMR_status != 0) {
2451         print STDERR "ERROR: ImR Service returned $IMR_status\n";
2452         return 1;
2453     }
2454
2455     if (wait_for_imr(\%backup_repo, "replicaiorfile")) {
2456         return 1;
2457     }
2458
2459     print "\n\nstarting backup tao_imr_locator\n";
2460     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
2461         "$backup_repo{imriorfile} $imr_refstyle " .
2462         "$backup_repo{imr_endpoint_flag} " .
2463         "$backup_repo{imr_backing_store_flag}");
2464     my $replica_IMR_status = $backup_repo{IMR}->Spawn ();
2465     if ($replica_IMR_status != 0) {
2466         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
2467         return 1;
2468     }
2469     if (wait_for_imr(\%repo, "replicaiorfile")) {
2470         return 1;
2471     }
2472     print "started backup tao_imr_locator\n";
2473
2474     if (wait_for_imr(\%repo)) {
2475         return 1;
2476     }
2477     print "started primary tao_imr_locator\n";
2478
2479     for ($index = 0; $index < $num_srvr; ++$index) {
2480         if ($a_srv[$index]->PutFile ($repo_for_srvr[$index]->{imriorfile}) == -1) {
2481             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
2482             kill_imr();
2483             return 1;
2484         }
2485     }
2486
2487     print "\n\nstarting tao_imr_activator\n";
2488     $repo{ACT}->Arguments ("-d $test_debug_level -o $repo{act_actiorfile} " .
2489         "-ORBInitRef ImplRepoService=file://$repo{act_imriorfile} $refstyle " .
2490         $repo{act_explicit_flag});
2491     my $ACT_status = $repo{ACT}->Spawn ();
2492     if ($ACT_status != 0) {
2493         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
2494         return 1;
2495     }
2496
2497     if ($repo{act}->WaitForFileTimed (
2498                   $repo{actiorfile},
2499                   $repo{act}->ProcessStartWaitInterval()) == -1) {
2500         print STDERR "ERROR: cannot find file <" .
2501             $repo{act_actiorfile} . ">\n";
2502         kill_act();
2503         kill_imr();
2504         return 1;
2505     }
2506     print "started tao_imr_activator\n";
2507
2508     print "Adding servers\n";
2509     if (add_servers (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2510         return 1;
2511     }
2512 #    for ($index = 0; $index < $num_srvr; ++$index) {
2513 #        print "\n\nadding server $index using tao_imr\n";
2514 #        $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
2515 #            "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
2516 #            "add $a_srv_name[$index] -c \"$imr_A_SRV_cmd[$index] " .
2517 #            "-o $imr_airplaneiorfile[$index] -s $a_srv_name[$index]\" " .
2518 #            "$repo_for_srvr[$index]->{server_act_flag} ");
2519 #        my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
2520 #            $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
2521 #        if ($TI_status != 0) {
2522 #            print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
2523 #            kill_act();
2524 #            kill_imr();
2525 #            return 1;
2526 #        }
2527 #        print "added server $index with the locator using tao_imr\n";
2528 #    }
2529
2530     if (add_servers_again (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2531         return 1;
2532     }
2533
2534     for ($index = 0; $index < $num_srvr; ++$index) {
2535         print "\n\nstarting server $index using tao_imr\n";
2536         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
2537             "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
2538             "start $a_srv_name[$index]");
2539         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
2540             $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
2541         if ($TI_status != 0) {
2542             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
2543             kill_act();
2544             kill_imr();
2545             return 1;
2546         }
2547         print "started server $index using tao_imr\n";
2548     }
2549
2550     for ($index = 0; $index < $num_srvr; ++$index) {
2551         if ($a_srv[$index]->WaitForFileTimed (
2552                   $airplaneiorfile[$index],
2553                   $a_srv[$index]->ProcessStartWaitInterval()) == -1) {
2554             print STDERR
2555               "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
2556             kill_then_timed_wait(\@A_SRV, 1);
2557             $ACT->Kill (); $ACT->TimedWait (1);
2558             $IMR->Kill (); $IMR->TimedWait (1);
2559             return 1;
2560         }
2561         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
2562             print STDERR
2563               "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
2564             kill_then_timed_wait(\@A_SRV, 1);
2565             $ACT->Kill (); $ACT->TimedWait (1);
2566             $IMR->Kill (); $IMR->TimedWait (1);
2567             return 1;
2568         }
2569         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
2570             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
2571             kill_then_timed_wait(\@A_SRV, 1);
2572             $ACT->Kill (); $ACT->TimedWait (1);
2573             $IMR->Kill (); $IMR->TimedWait (1);
2574             return 1;
2575         }
2576     }
2577
2578     if (start_clients() != 0) {
2579         return 1;
2580     }
2581
2582     if (stop_clients() != 0) {
2583         return 1;
2584     }
2585
2586     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
2587         return 1;
2588     }
2589
2590     if (start_clients() != 0) {
2591         return 1;
2592     }
2593
2594     if (stop_clients() != 0) {
2595         return 1;
2596     }
2597
2598     print "\n\nkilling the primary tao_imr_locator\n";
2599     $IMR->Kill(); $IMR->TimedWait();
2600     print "killed the primary tao_imr_locator\n";
2601
2602     if (start_clients() != 0) {
2603         return 1;
2604     }
2605
2606     if (stop_clients() != 0) {
2607         return 1;
2608     }
2609
2610     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
2611         return 1;
2612     }
2613
2614     if (add_servers_again(\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2615         return 1;
2616     }
2617
2618     if (remove_servers(\@repo_for_srvr) != 0) {
2619         return 1;
2620     }
2621
2622     if (add_servers(\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2623         return 1;
2624     }
2625
2626     print "\n\nstarting primary tao_imr_locator again\n";
2627     $imr->DeleteFile ($imriorfile);
2628     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
2629         "$imr_refstyle $repo{imr_endpoint_flag} $repo{imr_backing_store_flag}");
2630     $IMR_status = $repo{IMR}->Spawn ();
2631     if ($IMR_status != 0) {
2632         print STDERR "ERROR: ImR Service returned $IMR_status\n";
2633         return 1;
2634     }
2635     if (wait_for_imr(\%repo)) {
2636         return 1;
2637     }
2638     print "started primary tao_imr_locator again\n";
2639
2640     if (start_clients() != 0) {
2641         return 1;
2642     }
2643
2644     if (stop_clients() != 0) {
2645         return 1;
2646     }
2647
2648     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
2649         return 1;
2650     }
2651
2652     print "\n\nkilling the backup tao_imr_locator\n";
2653     $replica_IMR->Kill(); $replica_IMR->TimedWait();
2654     $replica_imr->DeleteFile ($replica_imriorfile);
2655     print "killed the backup tao_imr_locator\n";
2656
2657     print "\n\nstarting backup tao_imr_locator\n";
2658     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
2659         "$backup_repo{imriorfile} $imr_refstyle " .
2660         "$backup_repo{imr_endpoint_flag} " .
2661         "$backup_repo{imr_backing_store_flag}");
2662     $replica_IMR_status = $backup_repo{IMR}->Spawn ();
2663     if ($replica_IMR_status != 0) {
2664         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
2665         return 1;
2666     }
2667     if (wait_for_imr(\%repo, "replicaiorfile")) {
2668         return 1;
2669     }
2670     print "started backup tao_imr_locator again\n";
2671
2672     if (start_clients() != 0) {
2673         return 1;
2674     }
2675
2676     if (stop_clients() != 0) {
2677         return 1;
2678     }
2679
2680     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
2681         return 1;
2682     }
2683
2684     $IMR->Kill(); $IMR->TimedWait();
2685     $replica_IMR->Kill(); $replica_IMR->TimedWait();
2686     $ACT->Kill(); $ACT->TimedWait();
2687
2688     cleanup_replication ();
2689
2690     # clean up IOR files
2691     for ($index = 0; $index < $num_srvr; ++$index) {
2692         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
2693         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2694         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2695         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
2696     }
2697     $imr->DeleteFile ($imriorfile);
2698     $imr->DeleteFile ($primaryiorfile);
2699     $act->DeleteFile ($imriorfile);
2700     $ti->DeleteFile ($imriorfile);
2701     $act->DeleteFile ($actiorfile);
2702     $replica_imr->DeleteFile ($replica_imriorfile);
2703     $replica_imr->DeleteFile ($backupiorfile);
2704
2705     return $status;
2706 }
2707
2708 ###############################################################################
2709 # Start the primary then the backup. Kill both the primary and the backup and
2710 # restart the backup and verify it works.
2711
2712 sub backup_restart_test
2713 {
2714     if (!$replica) {
2715       # The failover test needs the -replica flag
2716       return 0;
2717     }
2718
2719     my $status = 0;
2720
2721     my $imr_port = 10001 + $imr->RandomPort ();
2722     my $replica_imr_port = $imr_port + 10;
2723
2724     my %repo;
2725     setup_repo(\%repo, $imr, $IMR, $imriorfile, $act, $ACT, $actiorfile, $ti,
2726       $TI, $imr_port, "--primary", $backupiorfile);
2727
2728     my %backup_repo;
2729     setup_repo(\%backup_repo, $replica_imr, $replica_IMR, $imriorfile,
2730       $act, $ACT, $actiorfile, $ti, $TI,
2731       $replica_imr_port, "--backup", $primaryiorfile);
2732
2733     my @repo_for_srvr;
2734     my @a_srv_imriorfile;
2735     my @imr_airplaneiorfile;
2736     my @a_srv_airplaneiorfile;
2737     for ($index = 0; $index < $num_srvr; ++$index) {
2738         if ($index == ($num_srvr - 1)) {
2739             push(@repo_for_srvr, \%backup_repo);
2740         } else {
2741             push(@repo_for_srvr, \%repo);
2742         }
2743
2744         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($repo_for_srvr[$index]->{imriorfile}));
2745         push(@imr_airplaneiorfile, $repo_for_srvr[$index]->{imr}->LocalFile ($airplaneiorfile[$index]));
2746         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
2747         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
2748         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2749         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2750         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
2751     }
2752     $imr->DeleteFile ($imriorfile);
2753     $imr->DeleteFile ($primaryiorfile);
2754     $act->DeleteFile ($imriorfile);
2755     $ti->DeleteFile ($imriorfile);
2756     $act->DeleteFile ($actiorfile);
2757     $replica_imr->DeleteFile ($replica_imriorfile);
2758     $replica_imr->DeleteFile ($backupiorfile);
2759
2760     cleanup_replication ();
2761
2762     print "\n\nstarting primary tao_imr_locator\n";
2763     print STDERR "tao_imr_locator -d $test_debug_level -o $repo{imr_imriorfile} $repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} \n";
2764
2765     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
2766         "$repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} ");
2767     my $IMR_status = $repo{IMR}->Spawn ();
2768     if ($IMR_status != 0) {
2769         print STDERR "ERROR: ImR Service returned $IMR_status\n";
2770         return 1;
2771     }
2772
2773     if (wait_for_imr(\%backup_repo, "replicaiorfile")) {
2774         return 1;
2775     }
2776
2777     print "\n\nstarting backup tao_imr_locator\n";
2778     print STDERR "-d $test_debug_level -o $backup_repo{imriorfile} $imr_refstyle $backup_repo{imr_endpoint_flag} $backup_repo{imr_backing_store_flag}\n";
2779
2780     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
2781         "$backup_repo{imriorfile} $imr_refstyle " .
2782         "$backup_repo{imr_endpoint_flag} " .
2783         "$backup_repo{imr_backing_store_flag}");
2784     my $replica_IMR_status = $backup_repo{IMR}->Spawn ();
2785     if ($replica_IMR_status != 0) {
2786         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
2787         return 1;
2788     }
2789     if (wait_for_imr(\%repo, "replicaiorfile")) {
2790         return 1;
2791     }
2792     print "started backup tao_imr_locator\n";
2793
2794     if (wait_for_imr(\%repo)) {
2795         return 1;
2796     }
2797     print "started primary tao_imr_locator\n";
2798
2799     for ($index = 0; $index < $num_srvr; ++$index) {
2800         if ($a_srv[$index]->PutFile ($repo_for_srvr[$index]->{imriorfile}) == -1) {
2801             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
2802             kill_imr();
2803             return 1;
2804         }
2805     }
2806
2807     print "\n\nstarting tao_imr_activator\n";
2808     $repo{ACT}->Arguments ("-d $test_debug_level -o $repo{act_actiorfile} " .
2809         "-ORBInitRef ImplRepoService=file://$repo{act_imriorfile} $refstyle " .
2810         $repo{act_explicit_flag});
2811     my $ACT_status = $repo{ACT}->Spawn ();
2812     if ($ACT_status != 0) {
2813         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
2814         return 1;
2815     }
2816
2817     if ($repo{act}->WaitForFileTimed (
2818                   $repo{actiorfile},
2819                   $repo{act}->ProcessStartWaitInterval()) == -1) {
2820         print STDERR "ERROR: cannot find file <" .
2821             $repo{act_actiorfile} . ">\n";
2822         kill_act();
2823         kill_imr();
2824         return 1;
2825     }
2826     print "started tao_imr_activator\n";
2827
2828     print "Adding servers\n";
2829     if (add_servers (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2830         return 1;
2831     }
2832
2833     if (add_servers_again (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
2834         return 1;
2835     }
2836
2837    for ($index = 0; $index < $num_srvr; ++$index) {
2838         print "\n\nstarting server $index using tao_imr\n";
2839         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
2840             "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
2841             "start $a_srv_name[$index]");
2842         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
2843             $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
2844         if ($TI_status != 0) {
2845             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
2846             kill_act();
2847             kill_imr();
2848             return 1;
2849         }
2850         print "started server $index using tao_imr\n";
2851     }
2852
2853     for ($index = 0; $index < $num_srvr; ++$index) {
2854         if ($a_srv[$index]->WaitForFileTimed (
2855                   $airplaneiorfile[$index],
2856                   $a_srv[$index]->ProcessStartWaitInterval()) == -1) {
2857             print STDERR
2858               "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
2859             kill_then_timed_wait(\@A_SRV, 1);
2860             $ACT->Kill (); $ACT->TimedWait (1);
2861             $IMR->Kill (); $IMR->TimedWait (1);
2862             return 1;
2863         }
2864         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
2865             print STDERR
2866               "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
2867             kill_then_timed_wait(\@A_SRV, 1);
2868             $ACT->Kill (); $ACT->TimedWait (1);
2869             $IMR->Kill (); $IMR->TimedWait (1);
2870             return 1;
2871         }
2872         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
2873             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
2874             kill_then_timed_wait(\@A_SRV, 1);
2875             $ACT->Kill (); $ACT->TimedWait (1);
2876             $IMR->Kill (); $IMR->TimedWait (1);
2877             return 1;
2878         }
2879     }
2880
2881     if (start_clients() != 0) {
2882         return 1;
2883     }
2884
2885     if (stop_clients() != 0) {
2886         return 1;
2887     }
2888
2889     if (shutdown_servers_using_ior($imriorfile, $ti) != 0) {
2890         return 1;
2891     }
2892
2893     if (start_clients() != 0) {
2894         return 1;
2895     }
2896
2897     if (stop_clients() != 0) {
2898         return 1;
2899     }
2900
2901     print "\n\nkilling the primary tao_imr_locator\n";
2902     $IMR->Kill(); $IMR->TimedWait();
2903     print "killed the primary tao_imr_locator\n";
2904
2905
2906     # Kill the backup and then restart it
2907     print "\n\nkilling the backup tao_imr_locator\n";
2908     $replica_IMR->Kill(); $replica_IMR->TimedWait();
2909     print "killed the backup tao_imr_locator\n";
2910
2911     print "\n\nstarting backup tao_imr_locator\n";
2912     print "tao_imr_locator -d $test_debug_level -o " .
2913         "$backup_repo{imriorfile} $imr_refstyle " .
2914         "$backup_repo{imr_endpoint_flag} " .
2915         "$backup_repo{imr_backing_store_flag} \n";
2916     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
2917         "$backup_repo{imriorfile} $imr_refstyle " .
2918         "$backup_repo{imr_endpoint_flag} " .
2919         "$backup_repo{imr_backing_store_flag}");
2920     $replica_IMR_status = $backup_repo{IMR}->Spawn ();
2921     if ($replica_IMR_status != 0) {
2922         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
2923         return 1;
2924     }
2925     if (wait_for_imr(\%repo, "replicaiorfile")) {
2926         return 1;
2927     }
2928     print "started backup tao_imr_locator again\n";
2929
2930     # Wait for the backup to restart.
2931     sleep (10);
2932
2933     if (start_clients() != 0) {
2934         return 1;
2935     }
2936
2937     if (stop_clients() != 0) {
2938         return 1;
2939     }
2940
2941     print STDERR "Shuting down servers after backup restarted\n";
2942     if (shutdown_servers_using_ior($imriorfile, $ti) != 0) {
2943         return 1;
2944     }
2945
2946     print "\n\nkilling the backup tao_imr_locator\n";
2947     $replica_IMR->Kill(); $replica_IMR->TimedWait();
2948     print "killed the backup tao_imr_locator\n";
2949
2950     # kill the activator
2951     $ACT->Kill(); $ACT->TimedWait();
2952
2953     cleanup_replication ();
2954
2955     # clean up IOR files
2956     for ($index = 0; $index < $num_srvr; ++$index) {
2957         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
2958         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
2959         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
2960         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
2961     }
2962     $imr->DeleteFile ($imriorfile);
2963     $imr->DeleteFile ($primaryiorfile);
2964     $act->DeleteFile ($imriorfile);
2965     $ti->DeleteFile ($imriorfile);
2966     $act->DeleteFile ($actiorfile);
2967     $replica_imr->DeleteFile ($replica_imriorfile);
2968     $replica_imr->DeleteFile ($backupiorfile);
2969
2970     return $status;
2971 }
2972
2973 ###############################################################################
2974
2975 sub persistent_ft_test
2976 {
2977     my $corrupted = shift;
2978
2979     if (!$corrupted) {
2980        $corrupted = 0;
2981     }
2982
2983     if (!$replica) {
2984       # The persistent_ft test needs the -replica flag
2985       return 0;
2986     }
2987
2988     my $status = 0;
2989
2990     my $imr_port = 10001 + $imr->RandomPort ();
2991     my $replica_imr_port = $imr_port + 1;
2992
2993     my %repo;
2994     setup_repo(\%repo, $imr, $IMR, $imriorfile, $act, $ACT, $actiorfile, $ti,
2995       $TI, $imr_port, "--primary", $backupiorfile);
2996
2997     my %backup_repo;
2998     setup_repo(\%backup_repo, $replica_imr, $replica_IMR, $replica_imriorfile,
2999       $act, $ACT, $actiorfile, $ti, $TI,
3000       $replica_imr_port, "--backup", $primaryiorfile);
3001
3002     my @repo_for_srvr;
3003     my @a_srv_imriorfile;
3004     my @imr_airplaneiorfile;
3005     my @a_srv_airplaneiorfile;
3006     for ($index = 0; $index < $num_srvr; ++$index) {
3007         if ($index == ($num_srvr - 1)) {
3008             push(@repo_for_srvr, \%backup_repo);
3009         } else {
3010             push(@repo_for_srvr, \%repo);
3011         }
3012
3013         push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($repo_for_srvr[$index]->{imriorfile}));
3014         push(@imr_airplaneiorfile, $repo_for_srvr[$index]->{imr}->LocalFile ($airplaneiorfile[$index]));
3015         push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
3016         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
3017         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
3018         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
3019         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
3020     }
3021     $imr->DeleteFile ($imriorfile);
3022     $imr->DeleteFile ($primaryiorfile);
3023     $act->DeleteFile ($imriorfile);
3024     $ti->DeleteFile ($imriorfile);
3025     $act->DeleteFile ($actiorfile);
3026     $replica_imr->DeleteFile ($replica_imriorfile);
3027     $replica_imr->DeleteFile ($backupiorfile);
3028
3029     print "\n\nstarting primary tao_imr_locator\n";
3030     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
3031         "$imr_refstyle $repo{imr_endpoint_flag} $repo{imr_backing_store_flag}");
3032 print "Comment line arguments: -d $test_debug_level -o $repo{imr_imriorfile} " .  "$imr_refstyle $repo{imr_endpoint_flag} $repo{imr_backing_store_flag}\n";
3033     my $IMR_status = $repo{IMR}->Spawn ();
3034     if ($IMR_status != 0) {
3035         print STDERR "ERROR: ImR Service returned $IMR_status\n";
3036         return 1;
3037     }
3038
3039     if (wait_for_imr(\%backup_repo, "replicaiorfile")) {
3040         return 1;
3041     }
3042
3043     print "\n\nstarting backup tao_imr_locator\n";
3044     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
3045         "$backup_repo{imriorfile} $imr_refstyle " .
3046         "$backup_repo{imr_endpoint_flag} " .
3047         "$backup_repo{imr_backing_store_flag}");
3048     my $replica_IMR_status = $backup_repo{IMR}->Spawn ();
3049     if ($replica_IMR_status != 0) {
3050         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
3051         return 1;
3052     }
3053     if (wait_for_imr(\%repo, "replicaiorfile")) {
3054         return 1;
3055     }
3056     print "started backup tao_imr_locator\n";
3057
3058     if (wait_for_imr(\%repo)) {
3059         return 1;
3060     }
3061     print "started primary tao_imr_locator\n";
3062
3063     for ($index = 0; $index < $num_srvr; ++$index) {
3064         if ($a_srv[$index]->PutFile ($repo_for_srvr[$index]->{imriorfile}) == -1) {
3065             print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
3066             kill_imr();
3067             return 1;
3068         }
3069     }
3070
3071     print "\n\nstarting tao_imr_activator\n";
3072     $repo{ACT}->Arguments ("-d $test_debug_level -o $repo{act_actiorfile} " .
3073         "-ORBInitRef ImplRepoService=file://$repo{act_imriorfile} $refstyle " .
3074         $repo{act_explicit_flag});
3075     my $ACT_status = $repo{ACT}->Spawn ();
3076     if ($ACT_status != 0) {
3077         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
3078         return 1;
3079     }
3080
3081     if ($repo{act}->WaitForFileTimed (
3082                   $repo{actiorfile},
3083                   $repo{act}->ProcessStartWaitInterval()) == -1) {
3084         print STDERR "ERROR: cannot find file <" .
3085             $repo{act_actiorfile} . ">\n";
3086         kill_act();
3087         kill_imr();
3088         return 1;
3089     }
3090     print "started tao_imr_activator\n";
3091
3092     for ($index = 0; $index < $num_srvr; ++$index) {
3093         print "\n\nupdating server $index using tao_imr\n";
3094         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
3095             "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
3096             "update $a_srv_name[$index] -c \"$imr_A_SRV_cmd[$index] " .
3097             "-o $imr_airplaneiorfile[$index] -s $a_srv_name[$index]\" " .
3098             "$repo_for_srvr[$index]->{server_act_flag} ");
3099         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
3100             $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
3101         if ($TI_status != 0) {
3102             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
3103             kill_act();
3104             kill_imr();
3105             return 1;
3106         }
3107         print "updated server $index with the locator using tao_imr\n";
3108     }
3109
3110     for ($index = 0; $index < $num_srvr; ++$index) {
3111         print "\n\nstarting server $index using tao_imr\n";
3112         $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
3113             "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
3114             "start $a_srv_name[$index]");
3115         my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
3116             $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
3117         if ($TI_status != 0) {
3118             print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
3119             kill_act();
3120             kill_imr();
3121             return 1;
3122         }
3123         print "started server $index using tao_imr\n";
3124     }
3125
3126     for ($index = 0; $index < $num_srvr; ++$index) {
3127         if ($a_srv[$index]->WaitForFileTimed (
3128                   $airplaneiorfile[$index],
3129                   $a_srv[$index]->ProcessStartWaitInterval()) == -1) {
3130             print STDERR
3131               "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
3132             kill_then_timed_wait(\@A_SRV, 1);
3133             $ACT->Kill (); $ACT->TimedWait (1);
3134             $IMR->Kill (); $IMR->TimedWait (1);
3135             return 1;
3136         }
3137         if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
3138             print STDERR
3139               "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
3140             kill_then_timed_wait(\@A_SRV, 1);
3141             $ACT->Kill (); $ACT->TimedWait (1);
3142             $IMR->Kill (); $IMR->TimedWait (1);
3143             return 1;
3144         }
3145         if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
3146             print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
3147             kill_then_timed_wait(\@A_SRV, 1);
3148             $ACT->Kill (); $ACT->TimedWait (1);
3149             $IMR->Kill (); $IMR->TimedWait (1);
3150             return 1;
3151         }
3152     }
3153
3154     if (start_clients() != 0) {
3155         return 1;
3156     }
3157
3158     if (stop_clients() != 0) {
3159         return 1;
3160     }
3161
3162     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
3163         return 1;
3164     }
3165
3166     print "\n\nkilling the primary tao_imr_locator\n";
3167     $IMR->Kill(); $IMR->TimedWait();
3168     $imr->DeleteFile ($imriorfile);
3169     $imr->DeleteFile ($primaryiorfile);
3170     print "killed the primary tao_imr_locator\n";
3171
3172     print "\n\nkilling the backup tao_imr_locator\n";
3173     $replica_IMR->Kill(); $replica_IMR->TimedWait();
3174     $replica_imr->DeleteFile ($replica_imriorfile);
3175     $replica_imr->DeleteFile ($backupiorfile);
3176     print "killed the backup tao_imr_locator\n";
3177
3178     if($corrupted == 1){
3179       my $file= "./imr_listing.xml";
3180       if (open LIST_FILE, ">$file") {
3181          print LIST_FILE "I'm corrupt!!!";
3182       }
3183     } elsif ($corrupted > 1) {
3184       my $file = "./1_" .($corrupted -1). ".xml";
3185       if (open LIST_FILE, ">$file") {
3186          print LIST_FILE "I'm corrupt!!!";
3187       }
3188     }
3189
3190     print "\n\nstarting primary tao_imr_locator again\n";
3191     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
3192         "$imr_refstyle $repo{imr_endpoint_flag} $repo{imr_backing_store_flag}");
3193
3194     my $test_stdout_file = $imr->LocalFile ($stdout_file);
3195     my $test_stderr_file = $imr->LocalFile ($stderr_file);
3196
3197     $imr->DeleteFile ($stdout_file);
3198     $imr->DeleteFile ($stderr_file);
3199
3200 #   Rely on return value only, so redirect output
3201     redirect_output($test_stdout_file,$test_stderr_file);
3202     $IMR_status = $repo{IMR}->Spawn ();
3203     restore_output();
3204
3205     if ($IMR_status != 0) {
3206         print STDERR "ERROR: ImR Service returned $IMR_status\n";
3207         return 1;
3208     }
3209
3210     if ($repo{imr}->WaitForFileTimed($primaryiorfile,
3211             $repo{imr}->ProcessStartWaitInterval()) == -1) {
3212         return 1;
3213     }
3214
3215     print "\n\nstarting backup tao_imr_locator again\n";
3216     $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
3217         "$backup_repo{imriorfile} $imr_refstyle " .
3218         "$backup_repo{imr_endpoint_flag} " .
3219         "$backup_repo{imr_backing_store_flag}");
3220
3221     redirect_output($test_stdout_file,$test_stderr_file);
3222     $replica_IMR_status = $backup_repo{IMR}->Spawn ();
3223     restore_output();
3224
3225     if ($replica_IMR_status != 0) {
3226         print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
3227         return 1;
3228     }
3229     if ($backup_repo{imr}->WaitForFileTimed($backupiorfile,
3230             $backup_repo{imr}->ProcessStartWaitInterval()) == -1) {
3231         return 1;
3232     }
3233     print "started backup tao_imr_locator again\n";
3234
3235     if (start_clients() != 0) {
3236         return 1;
3237     }
3238
3239     if (stop_clients() != 0) {
3240         return 1;
3241     }
3242
3243     if (shutdown_servers_using_tao_imr(\@repo_for_srvr) != 0) {
3244         return 1;
3245     }
3246
3247     $IMR->Kill(); $IMR->TimedWait();
3248     $replica_IMR->Kill(); $replica_IMR->TimedWait();
3249     $ACT->Kill(); $ACT->TimedWait();
3250
3251     cleanup_replication ();
3252
3253     # clean up IOR files
3254     for ($index = 0; $index < $num_srvr; ++$index) {
3255         $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
3256         $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
3257         $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
3258         $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
3259     }
3260     $imr->DeleteFile ($imriorfile);
3261     $imr->DeleteFile ($primaryiorfile);
3262     $act->DeleteFile ($imriorfile);
3263     $ti->DeleteFile ($imriorfile);
3264     $act->DeleteFile ($actiorfile);
3265     $replica_imr->DeleteFile ($replica_imriorfile);
3266     $replica_imr->DeleteFile ($backupiorfile);
3267
3268     return $status;
3269 }
3270
3271 ###############################################################################
3272
3273 sub get_server_pid
3274 {
3275     my $file = shift;
3276     my $pid = 0;
3277     my $server_pid = 0;
3278     open (FILE, $file) or die "Can't open $file: $!";
3279     while (<FILE>) {
3280         chomp;
3281         $pid = $_;
3282         $server_pid = $pid if ($server_pid == 0);
3283     }
3284     close FILE;
3285     return $server_pid;
3286 }
3287
3288 sub signal_server
3289 {
3290     my $pid = shift;
3291     my $sig = shift;
3292     print "signal $sig to server $pid\n";
3293     kill ($sig, $pid);
3294 }
3295
3296 sub both_ir_test
3297 {
3298     my $status = 0;
3299
3300     my $imr_imriorfile = $imr->LocalFile ($imriorfile);
3301     my $act_imriorfile = $act->LocalFile ($imriorfile);
3302     my $ti_imriorfile = $ti->LocalFile ($imriorfile);
3303     my $a_srv_imriorfile = $a_srv[0]->LocalFile ($imriorfile);
3304     my $n_srv_imriorfile = $n_srv->LocalFile ($imriorfile);
3305     my $act_actiorfile = $act->LocalFile ($actiorfile);
3306     my $a_srv_airplaneiorfile = $a_srv[0]->LocalFile ($airplaneiorfile[0]);
3307     my $n_srv_nesteaiorfile = $n_srv->LocalFile ($nesteaiorfile);
3308     my $n_cli_nesteaiorfile = $n_cli->LocalFile ($nesteaiorfile);
3309     $imr->DeleteFile ($imriorfile);
3310     $act->DeleteFile ($imriorfile);
3311     $ti->DeleteFile ($imriorfile);
3312     $a_srv[0]->DeleteFile ($imriorfile);
3313     $n_srv->DeleteFile ($imriorfile);
3314     $act->DeleteFile ($actiorfile);
3315     $a_srv[0]->DeleteFile ($airplaneiorfile[0]);
3316     $a_cli[0]->DeleteFile ($airplaneiorfile[0]);
3317     $n_srv->DeleteFile ($nesteaiorfile);
3318     $n_cli->DeleteFile ($nesteaiorfile);
3319
3320     $IMR->Arguments ("-d $test_debug_level -t 10 -o $imr_imriorfile $imr_refstyle");
3321     my $IMR_status = $IMR->Spawn ();
3322     if ($IMR_status != 0) {
3323         print STDERR "ERROR: ImR Service returned $IMR_status\n";
3324         return 1;
3325     }
3326     if ($imr->WaitForFileTimed ($imriorfile,
3327                                 $imr->ProcessStartWaitInterval()) == -1) {
3328         print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
3329         $IMR->Kill (); $IMR->TimedWait (1);
3330         return 1;
3331     }
3332     if ($imr->GetFile ($imriorfile) == -1) {
3333         print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
3334         $IMR->Kill (); $IMR->TimedWait (1);
3335         return 1;
3336     }
3337     if ($act->PutFile ($imriorfile) == -1) {
3338         print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
3339         $IMR->Kill (); $IMR->TimedWait (1);
3340         return 1;
3341     }
3342     if ($ti->PutFile ($imriorfile) == -1) {
3343         print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
3344         $IMR->Kill (); $IMR->TimedWait (1);
3345         return 1;
3346     }
3347     if ($a_srv[0]->PutFile ($imriorfile) == -1) {
3348         print STDERR "ERROR: cannot set file <$a_srv_imriorfile>\n";
3349         $IMR->Kill (); $IMR->TimedWait (1);
3350         return 1;
3351     }
3352     if ($n_srv->PutFile ($imriorfile) == -1) {
3353         print STDERR "ERROR: cannot set file <$n_srv_imriorfile>\n";
3354         $IMR->Kill (); $IMR->TimedWait (1);
3355         return 1;
3356     }
3357
3358     $ACT->Arguments ("-e 16384 -o $act_actiorfile ".
3359                      "-ORBInitRef ImplRepoService=file://$act_imriorfile ".
3360                      "$refstyle -d $test_debug_level");
3361     my $ACT_status = $ACT->Spawn ();
3362     if ($ACT_status != 0) {
3363         print STDERR "ERROR: ImR Activator returned $ACT_status\n";
3364         return 1;
3365     }
3366     if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
3367         print STDERR "ERROR: cannot find file <$act_actiorfile>\n";
3368         $ACT->Kill (); $ACT->TimedWait (1);
3369         $IMR->Kill (); $IMR->TimedWait (1);
3370         return 1;
3371     }
3372
3373     ## Note : It's crucial NOT to write out an IOR file when the activator
3374     ## starts the server, or at least to write out a different file name
3375     ## than the IOR files we're using for the clients. Otherwise a client
3376     ## may attempt to use a partially written file.
3377     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
3378     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
3379                     "add nestea_server -c \"$imr_N_SRV_cmd $refstyle\"");
3380     my $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3381     if ($TI_status != 0) {
3382         print STDERR "ERROR: tao_imr returned $TI_status\n";
3383         $ACT->Kill (); $ACT->TimedWait (1);
3384         $IMR->Kill (); $IMR->TimedWait (1);
3385         return 1;
3386     }
3387
3388     # No need to specify imr_initref or -orbuseimr 1 for servers spawned by activator
3389     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$ti_imriorfile ".
3390                     "add $a_srv_name[0] -c \"$imr_A_SRV_cmd[0] $refstyle -s $a_srv_name[0]\"");
3391     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3392     if ($TI_status != 0) {
3393         print STDERR "ERROR: tao_imr returned $TI_status\n";
3394         $ACT->Kill (); $ACT->TimedWait (1);
3395         $IMR->Kill (); $IMR->TimedWait (1);
3396         return 1;
3397     }
3398
3399     $N_SRV->Arguments (" -o $n_srv_nesteaiorfile -ORBUseIMR 1 ".
3400                        "-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3401                        "$refstyle ".
3402                        "-ORBDebugLevel $srv_debug_level");
3403     my $N_SRV_status = $N_SRV->Spawn ();
3404     if ($N_SRV_status != 0) {
3405         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
3406         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3407         $ACT->Kill (); $ACT->TimedWait (1);
3408         $IMR->Kill (); $IMR->TimedWait (1);
3409         return 1;
3410     }
3411     if ($n_srv->WaitForFileTimed ($nesteaiorfile,$n_srv->ProcessStartWaitInterval()) == -1) {
3412         print STDERR "ERROR: cannot find file <$n_srv_nesteaiorfile>\n";
3413         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3414         $ACT->Kill (); $ACT->TimedWait (1);
3415         $IMR->Kill (); $IMR->TimedWait (1);
3416         return 1;
3417     }
3418     if ($n_srv->GetFile ($nesteaiorfile) == -1) {
3419         print STDERR "ERROR: cannot retrieve file <$n_srv_nesteaiorfile>\n";
3420         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3421         $ACT->Kill (); $ACT->TimedWait (1);
3422         $IMR->Kill (); $IMR->TimedWait (1);
3423         return 1;
3424     }
3425     if ($n_cli->PutFile ($nesteaiorfile) == -1) {
3426         print STDERR "ERROR: cannot set file <$n_cli_nesteaiorfile>\n";
3427         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3428         $ACT->Kill (); $ACT->TimedWait (1);
3429         $IMR->Kill (); $IMR->TimedWait (1);
3430         return 1;
3431     }
3432
3433     $A_SRV[0]->Arguments (
3434       " -o $a_srv_airplaneiorfile -ORBUseIMR 1 ".
3435       "-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3436       "$refstyle -ORBDebugLevel $srv_debug_level -s $a_srv_name[0]");
3437     my $A_SRV_status = $A_SRV[0]->Spawn ();
3438     if ($A_SRV_status != 0) {
3439         print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
3440         kill_then_timed_wait(\@A_SRV, 1);
3441         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3442         $ACT->Kill (); $ACT->TimedWait (1);
3443         $IMR->Kill (); $IMR->TimedWait (1);
3444         return 1;
3445     }
3446     if ($a_srv[0]->WaitForFileTimed ($airplaneiorfile[0],$a_srv[0]->ProcessStartWaitInterval()) == -1) {
3447         print STDERR "ERROR: cannot find file <$a_srv_airplaneiorfile>\n";
3448         kill_then_timed_wait(\@A_SRV, 1);
3449         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3450         $ACT->Kill (); $ACT->TimedWait (1);
3451         $IMR->Kill (); $IMR->TimedWait (1);
3452         return 1;
3453     }
3454     if ($a_srv[0]->GetFile ($airplaneiorfile[0]) == -1) {
3455         print STDERR "ERROR: cannot retrieve file <$a_srv_airplaneiorfile>\n";
3456         kill_then_timed_wait(\@A_SRV, 1);
3457         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3458         $ACT->Kill (); $ACT->TimedWait (1);
3459         $IMR->Kill (); $IMR->TimedWait (1);
3460         return 1;
3461     }
3462     if ($a_cli[0]->PutFile ($airplaneiorfile[0]) == -1) {
3463         print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
3464         kill_then_timed_wait(\@A_SRV, 1);
3465         $N_SRV->Kill (); $N_SRV->TimedWait (1);
3466         $ACT->Kill (); $ACT->TimedWait (1);
3467         $IMR->Kill (); $IMR->TimedWait (1);
3468         return 1;
3469     }
3470
3471     my @a_clients;
3472     my @n_clients;
3473
3474     for (1 .. 5) {
3475        push @a_clients, &create_acli();
3476        push @n_clients, &create_ncli();
3477     }
3478
3479     print "\n## Spawning multiple simultaneous clients with both servers running.\n";
3480     map $_->Spawn(), @a_clients;
3481     map $_->Spawn(), @n_clients;
3482     map $_->WaitKill($a_cli[0]->ProcessStopWaitInterval()), @a_clients;
3483     map $_->WaitKill($n_cli->ProcessStopWaitInterval()), @n_clients;
3484
3485     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3486                     "shutdown nestea_server");
3487     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3488     if ($TI_status != 0) {
3489         print STDERR "ERROR: tao_imr returned $TI_status\n";
3490         $status = 1;
3491     }
3492
3493     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3494                     "shutdown $a_srv_name[0]");
3495     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3496     if ($TI_status != 0) {
3497         print STDERR "ERROR: tao_imr returned $TI_status\n";
3498         $status = 1;
3499     }
3500
3501     $A_SRV_status = $A_SRV[0]->WaitKill ($a_srv[0]->ProcessStopWaitInterval());
3502     if ($A_SRV_status != 0) {
3503         print STDERR "ERROR: Airplane Server returned $A_SRV_status\n";
3504         $status = 1;
3505     }
3506     $N_SRV_status = $N_SRV->WaitKill ($n_srv->ProcessStopWaitInterval());
3507     if ($N_SRV_status != 0) {
3508         print STDERR "ERROR: Nestea Server returned $N_SRV_status\n";
3509         $status = 1;
3510     }
3511
3512     print "\n\n\n\n## Spawning multiple simultaneous clients with no servers running.\n";
3513
3514     map $_->Spawn(), @a_clients;
3515     map $_->Spawn(), @n_clients;
3516     map $_->WaitKill($a_cli[0]->ProcessStopWaitInterval()), @a_clients;
3517     map $_->WaitKill($n_cli->ProcessStopWaitInterval()), @n_clients;
3518
3519     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3520                     "shutdown nestea_server");
3521     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3522     if ($TI_status != 0) {
3523         print STDERR "ERROR: tao_imr returned $TI_status\n";
3524         $status = 1;
3525     }
3526
3527     $TI->Arguments ("-ORBInitRef ImplRepoService=file://$n_srv_imriorfile ".
3528                     "shutdown $a_srv_name[0]");
3529     $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
3530     if ($TI_status != 0) {
3531         print STDERR "ERROR: tao_imr returned $TI_status\n";
3532         $status = 1;
3533     }
3534
3535     $ACT_status = $ACT->TerminateWaitKill ($act->ProcessStopWaitInterval());
3536     if ($ACT_status != 0) {
3537         print STDERR "ERROR: Activator returned $ACT_status\n";
3538         $status = 1;
3539     }
3540
3541     $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
3542     if ($IMR_status != 0) {
3543         print STDERR "ERROR: ImR returned $IMR_status\n";
3544         $status = 1;
3545     }
3546
3547     $imr->DeleteFile ($imriorfile);
3548     $act->DeleteFile ($imriorfile);
3549     $ti->DeleteFile ($imriorfile);
3550     $a_srv[0]->DeleteFile ($imriorfile);
3551     $n_srv->DeleteFile ($imriorfile);
3552     $act->DeleteFile ($actiorfile);
3553     $a_srv[0]->DeleteFile ($airplaneiorfile[0]);
3554     $a_cli[0]->DeleteFile ($airplaneiorfile[0]);
3555     $n_srv->DeleteFile ($nesteaiorfile);
3556     $n_cli->DeleteFile ($nesteaiorfile);
3557
3558     return $status;
3559 }
3560
3561 ###############################################################################
3562 ###############################################################################
3563
3564 # Parse the arguments
3565
3566 my $ret = 0;
3567 my $test_num = 0;
3568
3569 my @tests = ("airplane", "airplane_ir", "nestea", "nestea_ir",
3570              "both_ir", "persistent_ir", "persistent_ir_hash",
3571              "persistent_ir_shared", "persistent_ft", "failover",
3572              "backup_restart", "manual_persistent_restart",
3573              "manual_persistent_restart_hash",
3574              "manual_persistent_restart_shared");
3575
3576 my @nt_tests = ("nt_service_ir", "persistent_ir_registry", "manual_persistent_restart_registry");
3577
3578 my $i;
3579 if ($all_tests) {
3580     push(@ARGV, @tests);
3581 }
3582
3583 if ($#ARGV == -1) {
3584     $ret = both_ir_test();
3585 }
3586
3587 my $append = "";
3588 for ($i = 0; $i <= $#ARGV; $i++) {
3589     if ($ARGV[$i] eq "-h" || $ARGV[$i] eq "-?") {
3590         print "run_test [options] test\n";
3591         print "\n";
3592         print "options            \n";
3593         print "    -debug         -- set debug_level to 10\n";
3594         print "    -tdebug        -- set the tests debug level to 10\n";
3595         print "    -all           -- causes all tests to run (except nt_service_ir and\n";
3596         print "                      persistent_ir_registry)\n";
3597         print "    -servers       -- run tests with more than 1 server and client\n";
3598         print "    -replica       -- run tests with more than 1 server and client interacting\n";
3599         print "                      with replicated ImR Locators\n";
3600         print "test               -- Runs a specific test:\n";
3601         my $indent = "                         ";
3602         print "$indent";
3603         my $count = 0;
3604         my @all_tests = @tests;
3605         push(@all_tests, @nt_tests);
3606         foreach my $test (@all_tests) {
3607           if (++$count > 1) {
3608               print ",$append";
3609           }
3610           print "$test";
3611           $append = "";
3612           if ($count % 3 == 0) {
3613               $append = "\n$indent";
3614           }
3615         }
3616         print "\n";
3617         exit 1;
3618     }
3619     elsif ($ARGV[$i] eq "airplane") {
3620         $ret = airplane_test ();
3621     }
3622     elsif ($ARGV[$i] eq "airplane_ir") {
3623         $ret = airplane_ir_test ();
3624     }
3625     elsif ($ARGV[$i] eq "nt_service_ir") {
3626         $ret = nt_service_test ();
3627     }
3628     elsif ($ARGV[$i] eq "manual_persistent_restart") {
3629         $ret = manual_persistent_restart_test ("-x");
3630     }
3631     elsif ($ARGV[$i] eq "manual_persistent_restart_hash") {
3632         $ret = manual_persistent_restart_test ("-p");
3633     }
3634     elsif ($ARGV[$i] eq "manual_persistent_restart_registry") {
3635         $ret = manual_persistent_restart_test ("-x");
3636     }
3637     elsif ($ARGV[$i] eq "manual_persistent_restart_shared") {
3638         $ret = manual_persistent_restart_test ("--directory");
3639     }
3640     elsif ($ARGV[$i] eq "nestea") {
3641         $ret = nestea_test ();
3642     }
3643     elsif ($ARGV[$i] eq "nestea_ir") {
3644         $ret = nestea_ir_test ();
3645     }
3646     elsif ($ARGV[$i] eq "both_ir") {
3647         $ret = both_ir_test ();
3648     }
3649     elsif ($ARGV[$i] eq "persistent_ir") {
3650         $ret = persistent_ir_test ("-x");
3651     }
3652     elsif ($ARGV[$i] eq "persistent_ir_hash") {
3653         $ret = persistent_ir_test ("-p");
3654     }
3655     elsif ($ARGV[$i] eq "persistent_ir_registry") {
3656         $ret = persistent_ir_test ("-r");
3657     }
3658     elsif ($ARGV[$i] eq "persistent_ir_shared") {
3659         $ret = persistent_ir_test ("--directory");
3660     }
3661     elsif ($ARGV[$i] eq "persistent_ft") {
3662         $ret = persistent_ft_test ();
3663     }
3664     elsif ($ARGV[$i] eq "persistent_listingcorrupt") {
3665         $ret = persistent_ft_test (1);
3666     }
3667     elsif ($ARGV[$i] eq "persistent_activatorcorrupt") {
3668         $ret = persistent_ft_test (2);
3669     }
3670     elsif ($ARGV[$i] eq "persistent_servercorrupt") {
3671         $ret = persistent_ft_test (3);
3672     }
3673     elsif ($ARGV[$i] eq "failover") {
3674         $ret = failover_test ();
3675     }
3676     elsif ($ARGV[$i] eq "backup_restart") {
3677         $ret = backup_restart_test ();
3678     }
3679     elsif ($ARGV[$i] eq "perclient") {
3680         $ret = perclient();
3681     }
3682     elsif ($ARGV[$i] eq "shutdown") {
3683         $ret = shutdown_repo();
3684     }
3685     elsif ($ARGV[$i] !~ /^-/) {
3686         print "run_test: Unknown Option: ".$ARGV[$i]."\n";
3687     }
3688     if ($ret != 0) {
3689         last;
3690     }
3691 }
3692
3693 exit $ret;