TAO/orbsvcs/tests/FT_App/run_test_rmregistry.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2      & eval 'exec perl -S $0 $argv:q'
   3     if 0;
   4
   5 # -*- perl -*-
   6
   7 # Purpose:
   8 #       To test the FactoryRegistry as a component of ReplicationManager
   9 #
  10 # Process being tested:
  11 #       FT_ReplicationManager
  12 #           implements PortableGroup::FactoryRegistry interface.
  13 # Processes used in test:
  14 #       FT_Replica * 3
  15 #           implements GenericFactory interface to create TestReplicas
  16 #           TestReplica implements TestReplica interface.
  17 #           TestReplica implements PullMonitorable interface.
  18 #       FT_Client
  19 #           client for TestReplica interface.
  20 #           client for PullMonitorable.
  21 #       Object Group Creator
  22 #           Creates groups of objects.
  23 #
  24 # Test Scenario (***Test: marks behavior being tested):
  25 #
  26 #   First the test starts The ReplicationManager,
  27 #   Then starts three factories.  Each factory exists at a separate location.
  28 #   The locations are named shire, bree, and rivendell.  In a "real" system
  29 #   these locations would be on separate computers.
  30 #
  31 #   The factory at the shire location knows how to create hobbits.
  32 #   The factory at the bree location knows how to create hobbits, elves, and humans.
  33 #   The factory at rivendell can create elves.
  34 #   Hobbits, elves, and humans are roles for TestReplica objects.  A creation parameter
  35 #   included as part of the registration information specifies which role they are playing.
  36 #
  37 #   ***Test: The factories register themselves with the factory registry in the ReplicationManager.
  38 #   Registration information includes:
  39 #     the role,
  40 #     the type of object created,
  41 #     the location,
  42 #     an object reference to the factory and
  43 #     a set of parameters to be passed to the factory's create_object method.
  44 #
  45 #   An object group creator is started and asked to create three object groups:
  46 #   a group of hobbits, a group of elves; and another group of hobbits.
  47 #
  48 #   ***Test: the object group creator asks the ReplicationManager::FactoryRegistry
  49 #   for the set of factories that can create objects for the desired role.
  50 #   Then it uses the create_object method for each factory to create the actual object.
  51 #
  52 #   [temporary until IOGR's are working:  The object group creator writes the
  53 #   IOR's of the create objects to files -- using a file naming convention to
  54 #   distinguish members of the group.  It will be enhanced to assemble these
  55 #   IORs into an IOGR and either write the IOGR to a file or register it with
  56 #   the Naming Service.]
  57 #
  58 #   The object group creator is also told to unregister all factories that create humans.
  59 #   ***Test: It does so using the unregister_factory_by_role method.
  60 #
  61 #   Three clients are started, one at a time.  Each client is given a reference
  62 #   to an object group
  63 #
  64 #   [temporary until IOGRs and transparent reinvocaton work:  each client is
  65 #   given references to the members of the group and manages its own recovery
  66 #   (client mediated fault tolerance)]
  67 #
  68 #   Each client sends a few requests to first member of the object group.  Eventually
  69 #   this member fails and the requests are then rerouted to the second (and last)
  70 #   member of the group.
  71 #
  72 #   When a clients job is done, it exits and asks the remaining group member to
  73 #   exit, too.
  74 #
  75 #   The factories are run with the quit-on-idle option so when the last object
  76 #   created at that location goes away, the factory exits.
  77 #
  78 #   ***Test: As it exits the factory unregisters itself with the ReplicationManager::FactoryRegistry.
  79 #   ***Test: A command line option determines whether it uses a single
  80 #   unregister_factory_by_location call, or separate unregister_factory calles for
  81 #   each type of object created.  In this test, the shire location uses unregister_factory,
  82 #   and bree and rivendell use unregister_factory_by_location.
  83 #
  84 #   The factory registry is also run with the quit-on-idle option so when the last
  85 #   factory unregisters itself.
  86 #   The ReplicationManager is killed because it doesn't have a quit-on-idle option.
  87 use lib "$ENV{ACE_ROOT}/bin";
  88 use PerlACE::TestTarget;
  89
  90 ########################
  91 #command line options
  92 #set defaults:
  93
  94 my($status) = 0;
  95 my($verbose) = 0;         # 1: report perl actions before executing them
  96 my($debug_builds) = 0;    # 0: use exes from Release directories
  97 my($simulated) = 1;       # 1: use "client simulated" fault tolerance
  98
  99
 100 # List of the proccess which must be killed before exit with error
 101 @kill_list = ();
 102
 103 sub add_to_kills{
 104     $goal = shift;
 105     @kill_list = (@kill_list, $goal);
 106 }
 107 sub exit_and_kill{
 108     $status = shift;
 109     foreach $goal (@kill_list){
 110         $goal->Kill (); $goal->TimedWait (1);
 111     }
 112     exit $status;
 113 }
 114
 115 foreach $i (@ARGV) {
 116     if ($i eq "--debug_build"){
 117         $debug_builds = 1;
 118     }
 119     elsif ($i eq "--no_simulate"){  # reverse this once we have FT ORB support
 120         $simulated = 0;
 121     }
 122     elsif ($i eq "-v"){
 123         $verbose += 1;
 124     }
 125 }
 126
 127 my($build_directory) = "/Release";
 128 if ( $debug_builds ) {
 129     $build_directory = "";
 130 }
 131
 132 if ( $verbose > 1) {
 133     print "verbose: $verbose\n";
 134     print "debug_builds: $debug_builds -> $build_directory\n";
 135     print "simulated: $simulated\n";
 136 }
 137
 138 my($role1) = "hobbit";
 139 my($role2) = "elf";
 140 my($role3) = "human";
 141
 142 my($location1) = "shire";
 143 my($location2) = "bree";
 144 my($location3) = "rivendell";
 145 my($location4) = "rohan";
 146
 147 my $rp_manager = PerlACE::TestTarget::create_target (1) || die "Create target 1 failed\n";
 148 my $rp_manager_controller = PerlACE::TestTarget::create_target (2) || die "Create target 2 failed\n";
 149 my $ft_replica1 = PerlACE::TestTarget::create_target (3) || die "Create target 3 failed\n";
 150 my $ft_replica2 = PerlACE::TestTarget::create_target (4) || die "Create target 4 failed\n";
 151 my $ft_replica3 = PerlACE::TestTarget::create_target (5) || die "Create target 5 failed\n";
 152 my $ft_creator = PerlACE::TestTarget::create_target (6) || die "Create target 6 failed\n";
 153 my $client1 = PerlACE::TestTarget::create_target (7) || die "Create target 7 failed\n";
 154 my $client2 = PerlACE::TestTarget::create_target (8) || die "Create target 8 failed\n";
 155 my $client3 = PerlACE::TestTarget::create_target (9) || die "Create target 9 failed\n";
 156
 157 $hostname = $rp_manager->HostName ();
 158 $port = $rp_manager->RandomPort ();
 159
 160 #define temp files
 161 my($rm_ior) = "rm.ior";
 162
 163 my($factory1_ior) = "factory1.ior";
 164 my($factory2_ior) = "factory2.ior";
 165 my($factory3_ior) = "factory3.ior";
 166
 167 my($replica1_ior) = "${role1}_${location1}_0.ior";
 168 my($replica2_ior) = "${role1}_${location2}_0.ior";
 169 my($replica3_ior) = "${role2}_${location2}_1.ior";
 170 my($replica4_ior) = "${role2}_${location3}_0.ior";
 171 my($replica5_ior) = "${role1}_${location1}_1.ior";
 172 my($replica6_ior) = "${role1}_${location2}_2.ior";
 173
 174 my($replica1_iogr) = "${role1}_0.iogr";
 175 my($replica2_iogr) = "${role2}_1.iogr";
 176 my($replica3_iogr) = "${role1}_2.iogr";
 177
 178 my($client_data) = "persistent.dat";
 179
 180 # Files for rp_manager
 181 my $rp_manager_ior = $rp_manager->LocalFile ($rm_ior);
 182 $rp_manager->DeleteFile ($rm_ior);
 183
 184 # Files for ft_replica1
 185 my $ft_replica1_factory1_ior = $ft_replica1->LocalFile ($factory1_ior);
 186 my $ft_replica1_client_data = $ft_replica1->LocalFile ($client_data);
 187
 188 $ft_replica1->DeleteFile ($factory1_ior);
 189 $ft_replica1->DeleteFile ($client_data);
 190
 191 # Files for ft_replica2
 192 my $ft_replica2_factory2_ior = $ft_replica2->LocalFile ($factory2_ior);
 193 my $ft_replica2_client_data = $ft_replica2->LocalFile ($client_data);
 194
 195 $ft_replica2->DeleteFile ($factory2_ior);
 196 $ft_replica2->DeleteFile ($client_data);
 197
 198
 199 # Files for ft_replica3
 200 my $ft_replica3_factory3_ior = $ft_replica3->LocalFile ($factory3_ior);
 201 my $ft_replica3_client_data = $ft_replica3->LocalFile ($client_data);
 202
 203 $ft_replica3->DeleteFile ($factory3_ior);
 204 $ft_replica3->DeleteFile ($client_data);
 205
 206
 207 # Files for ft_creator
 208 my $creator_replica1_ior = $ft_creator->LocalFile ($replica1_ior);
 209 my $creator_replica2_ior = $ft_creator->LocalFile ($replica2_ior);
 210 my $creator_replica3_ior = $ft_creator->LocalFile ($replica3_ior);
 211 my $creator_replica4_ior = $ft_creator->LocalFile ($replica4_ior);
 212 my $creator_replica5_ior = $ft_creator->LocalFile ($replica5_ior);
 213 my $creator_replica6_ior = $ft_creator->LocalFile ($replica6_ior);
 214
 215 my $creator_replica1_iogr = $ft_creator->LocalFile ($replica1_iogr);
 216 my $creator_replica2_iogr = $ft_creator->LocalFile ($replica2_iogr);
 217 my $creator_replica3_iogr = $ft_creator->LocalFile ($replica3_iogr);
 218
 219
 220 $ft_creator->DeleteFile ($replica1_ior);
 221 $ft_creator->DeleteFile ($replica2_ior);
 222 $ft_creator->DeleteFile ($replica3_ior);
 223 $ft_creator->DeleteFile ($replica4_ior);
 224 $ft_creator->DeleteFile ($replica5_ior);
 225 $ft_creator->DeleteFile ($replica6_ior);
 226
 227 $ft_creator->DeleteFile ($replica1_iogr);
 228 $ft_creator->DeleteFile ($replica2_iogr);
 229 $ft_creator->DeleteFile ($replica3_iogr);
 230
 231 # Files for client1
 232 my $client1_replica1_ior = $client1->LocalFile ($replica1_ior);
 233 my $client1_replica2_ior = $client1->LocalFile ($replica2_ior);
 234 my $client1_replica1_iogr = $client1->LocalFile ($replica1_iogr);
 235
 236 $client1->DeleteFile ($replica1_ior);
 237 $client1->DeleteFile ($replica2_ior);
 238 $client1->DeleteFile ($replica1_iogr);
 239
 240 # Files for client2
 241 my $client2_replica3_ior = $client2->LocalFile ($replica3_ior);
 242 my $client2_replica4_ior = $client2->LocalFile ($replica4_ior);
 243 my $client2_replica2_iogr = $client2->LocalFile ($replica2_iogr);
 244
 245 $client2->DeleteFile ($replica3_ior);
 246 $client2->DeleteFile ($replica4_ior);
 247 $client2->DeleteFile ($replica2_iogr);
 248
 249 # Files for client3
 250 my $client3_replica5_ior = $client3->LocalFile ($replica5_ior);
 251 my $client3_replica6_ior = $client3->LocalFile ($replica6_ior);
 252 my $client3_replica3_iogr = $client3->LocalFile ($replica3_iogr);
 253
 254 $client3->DeleteFile ($replica5_ior);
 255 $client3->DeleteFile ($replica6_ior);
 256 $client3->DeleteFile ($replica3_iogr);
 257
 258 $RM = $rp_manager->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager" .
 259                                   "$build_directory/tao_ft_replicationmanager",
 260                                   "-o $rp_manager_ior " .
 261                                   "-ORBEndpoint iiop://$hostname:$port");
 262
 263 $RMC = $rp_manager_controller->CreateProcess (".$build_directory/replmgr_controller" ,
 264                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 265                                  "-x");
 266
 267 $REP1 = $ft_replica1->CreateProcess (".$build_directory/ft_replica" ,
 268                                  "-o $ft_replica1_factory1_ior ".
 269                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 270                                  "-l $location1 -i $role1 -q -p $ft_replica1_client_data");
 271
 272 $REP2 = $ft_replica2->CreateProcess (".$build_directory/ft_replica" ,
 273                                  "-o $ft_replica2_factory2_ior ".
 274                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 275                                  "-l $location2 ".
 276                                  "-i $role1 -i $role2 -i $role3 -q -u " .
 277                                  "-p $ft_replica2_client_data");
 278
 279 $REP3 = $ft_replica3->CreateProcess (".$build_directory/ft_replica" ,
 280                                  "-o $ft_replica3_factory3_ior ".
 281                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 282                                  "-l $location3 -i $role2 -q -u -p $ft_replica1_client_data");
 283
 284
 285 $CTR = $ft_creator->CreateProcess (".$build_directory/ft_create" ,
 286                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 287                                  "-n -r $role1 -r $role2 -r $role1 -u $role3 -i");
 288
 289
 290
 291 my($CL1) = $client1->CreateProcess (".$build_directory/ft_client" ,
 292                                  "-f file://$client1_replica1_ior " .
 293                                  "-f file://$client1_replica2_ior " .
 294                                  "-c testscript");
 295
 296 my($CL2) = $client2->CreateProcess (".$build_directory/ft_client" ,
 297                                  "-f file://$client2_replica3_ior " .
 298                                  "-f file://$client2_replica4_ior " .
 299                                  "-c testscript");
 300
 301 my($CL3) = $client3->CreateProcess (".$build_directory/ft_client" ,
 302                                  "-f file://$client3_replica5_ior " .
 303                                  "-f file://$client3_replica6_ior " .
 304                                  "-c testscript");
 305
 306
 307 if ($simulated) {
 308     print "\nTEST: Preparing Client Mediated Fault Tolerance test.\n" if ($verbose);
 309 }else{
 310     print "\nTEST: Preparing IOGR based test.\n" if ($verbose);
 311     $CL1->Arguments ("-f file://$client1_replica1_iogr -c testscript");
 312     $CL2->Arguments ("-f file://$client2_replica2_iogr -c testscript");
 313     $CL3->Arguments ("-f file://$client3_replica3_iogr -c testscript");
 314 }
 315
 316 #######################
 317 # ReplicationManager
 318
 319 print "\nTEST: starting ReplicationManager " . $RM->CommandLine . "\n" if ($verbose);
 320 $RM->Spawn ();
 321
 322 add_to_kills ($RM);
 323
 324 print "TEST: waiting for registry's IOR\n" if ($verbose);
 325
 326 if ($rp_manager->WaitForFileTimed ($rm_ior,
 327                                $rp_manager->ProcessStartWaitInterval()) == -1) {
 328     print STDERR "ERROR: cannot find file <$rp_manager_ior>\n";
 329     exit_and_kill (1);
 330 }
 331
 332 #################
 333 # Start Factories
 334
 335 print "\nTEST: starting factory 1 " . $REP1->CommandLine . "\n" if ($verbose);
 336 $REP1->Spawn ();
 337
 338 add_to_kills ($REP1);
 339
 340 print "TEST: waiting for factory 1's IOR\n" if ($verbose);
 341 if ($ft_replica1->WaitForFileTimed ($factory1_ior,
 342                                $ft_replica1->ProcessStartWaitInterval()) == -1) {
 343     print STDERR "ERROR: cannot find file <$ft_replica1_factory1_ior>\n";
 344     exit_and_kill (1);
 345 }
 346
 347 print "\nTEST: starting factory 2 " . $REP2->CommandLine . "\n" if ($verbose);
 348 $REP2->Spawn ();
 349
 350 add_to_kills ($REP2);
 351
 352 print "TEST: waiting for factory 2's IOR\n" if ($verbose);
 353 if ($ft_replica2->WaitForFileTimed ($factory2_ior,
 354                                $ft_replica2->ProcessStartWaitInterval()) == -1) {
 355     print STDERR "ERROR: cannot find file <$ft_replica2_factory2_ior>\n";
 356     exit_and_kill (1);
 357 }
 358
 359 print "\nTEST: starting factory 3 " . $REP3->CommandLine . "\n" if ($verbose);
 360 $REP3->Spawn ();
 361
 362 add_to_kills ($REP3);
 363
 364 print "TEST: waiting for factory 3's IOR\n" if ($verbose);
 365 if ($ft_replica3->WaitForFileTimed ($factory3_ior,
 366                                $ft_replica3->ProcessStartWaitInterval()) == -1) {
 367     print STDERR "ERROR: cannot find file <$ft_replica3_factory3_ior>\n";
 368     exit_and_kill (1);
 369 }
 370
 371 ######################
 372 # Create object groups
 373
 374 print "\nTEST: starting object group creator " . $CTR->CommandLine . "\n" if ($verbose);
 375 $CTR->Spawn ();
 376
 377 add_to_kills ($CTR);
 378
 379 print "TEST: waiting for Replica IOR files from object group creator\n" if ($verbose);
 380
 381 if ($ft_creator->WaitForFileTimed ($replica1_ior,
 382                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 383     print STDERR "ERROR: cannot find file <$ft_creator_replica1_ior>\n";
 384     exit_and_kill (1);
 385 }
 386
 387 if ($ft_creator->WaitForFileTimed ($replica2_ior,
 388                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 389     print STDERR "ERROR: cannot find file <$ft_creator_replica2_ior>\n";
 390     exit_and_kill (1);
 391 }
 392
 393 if ($ft_creator->WaitForFileTimed ($replica3_ior,
 394                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 395     print STDERR "ERROR: cannot find file <$ft_creator_replica3_ior>\n";
 396     exit_and_kill (1);
 397 }
 398
 399 if ($ft_creator->WaitForFileTimed ($replica4_ior,
 400                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 401     print STDERR "ERROR: cannot find file <$ft_creator_replica4_ior>\n";
 402     exit_and_kill (1);
 403 }
 404
 405 if ($ft_creator->WaitForFileTimed ($replica5_ior,
 406                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 407     print STDERR "ERROR: cannot find file <$ft_creator_replica5_ior>\n";
 408     exit_and_kill (1);
 409 }
 410
 411 if ($ft_creator->WaitForFileTimed ($replica6_ior,
 412                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 413     print STDERR "ERROR: cannot find file <$ft_creator_replica6_ior>\n";
 414     exit_and_kill (1);
 415 }
 416
 417 # Get/Put files:
 418 # replica1_ior: client1
 419 if ($ft_creator->GetFile ($replica1_ior) == -1) {
 420     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica1_ior\n";
 421     exit_and_kill (1);
 422 }
 423 if ($client1->PutFile ($replica1_ior) == -1) {
 424     print STDERR "ERROR: cannot set file <$client1_replica1_ior>\n";
 425     exit_and_kill (1);
 426 }
 427 # replica2_ior: client1
 428 if ($ft_creator->GetFile ($replica2_ior) == -1) {
 429     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica2_ior\n";
 430     exit_and_kill (1);
 431 }
 432 if ($client1->PutFile ($replica2_ior) == -1) {
 433     print STDERR "ERROR: cannot set file <$client1_replica2_ior>\n";
 434     exit_and_kill (1);
 435 }
 436 # replica3_ior: client2
 437 if ($ft_creator->GetFile ($replica3_ior) == -1) {
 438     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica3_ior\n";
 439     exit_and_kill (1);
 440 }
 441 if ($client2->PutFile ($replica3_ior) == -1) {
 442     print STDERR "ERROR: cannot set file <$client2_replica3_ior>\n";
 443     exit_and_kill (1);
 444 }
 445 # replica4_ior: client2
 446 if ($ft_creator->GetFile ($replica4_ior) == -1) {
 447     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica4_ior\n";
 448     exit_and_kill (1);
 449 }
 450 if ($client2->PutFile ($replica4_ior) == -1) {
 451     print STDERR "ERROR: cannot set file <$client2_replica4_ior>\n";
 452     exit_and_kill (1);
 453 }
 454 # replica5_ior: client3
 455 if ($ft_creator->GetFile ($replica5_ior) == -1) {
 456     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica5_ior\n";
 457     exit_and_kill (1);
 458 }
 459 if ($client3->PutFile ($replica5_ior) == -1) {
 460     print STDERR "ERROR: cannot set file <$client3_replica5_ior>\n";
 461     exit_and_kill (1);
 462 }
 463 # replica6_ior: client3
 464 if ($ft_creator->GetFile ($replica6_ior) == -1) {
 465     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica6_ior\n";
 466     exit_and_kill (1);
 467 }
 468 if ($client3->PutFile ($replica6_ior) == -1) {
 469     print STDERR "ERROR: cannot set file <$client3_replica6_ior>\n";
 470     exit_and_kill (1);
 471 }
 472
 473 print "\nTEST: wait for object group creator.\n" if ($verbose);
 474 $status_creator = $CTR->WaitKill ($ft_creator->ProcessStopWaitInterval());
 475
 476 if ($status_creator != 0) {
 477     print STDERR "TEST ERROR: configuration manager returned $status_creator\n";
 478     exit_and_kill(1);
 479 }
 480
 481 #############
 482 # Run clients
 483
 484 print "\nTEST: starting client " . $CL1->CommandLine . "\n" if ($verbose);
 485 $status_client = $CL1->SpawnWaitKill ($client1->ProcessStartWaitInterval() + 45);
 486 if ($status_client != 0) {
 487     print STDERR "TEST ERROR: client returned $status_client\n";
 488     exit_and_kill(1);
 489 }
 490
 491 print "\nTEST: starting client again " . $CL2->CommandLine . "\n" if ($verbose);
 492
 493 $status_client = $CL2->SpawnWaitKill ($client2->ProcessStartWaitInterval() + 45);
 494 if ($status_client != 0) {
 495     print STDERR "TEST ERROR: client returned $status_client\n";
 496     exit_and_kill(1);
 497 }
 498
 499 print "\nTEST: starting client, one more time with feeling " . $CL3->CommandLine . "\n" if ($verbose);
 500 $status_client = $CL3->SpawnWaitKill ($client3->ProcessStartWaitInterval() + 45);
 501 if ($status_client != 0) {
 502     print STDERR "TEST ERROR: client returned $status_client\n";
 503     exit_and_kill(1);
 504 }
 505
 506 ######################
 507 # Clean house and exit
 508
 509 print "\nTEST: wait for factory 1.\n" if ($verbose);
 510
 511 $status_replica = $REP1->WaitKill ($ft_replica1->ProcessStopWaitInterval() + 15);
 512 if ($status_replica != 0) {
 513     print STDERR "TEST ERROR: replica returned $status_replica\n";
 514     exit_and_kill(1);
 515 }
 516 print "\nTEST: wait for factory 2.\n" if ($verbose);
 517 $status_replica = $REP2->WaitKill ($ft_replica2->ProcessStopWaitInterval() + 15);
 518 if ($status_replica != 0) {
 519     print STDERR "TEST ERROR: replica returned $status_replica\n";
 520     exit_and_kill(1);
 521 }
 522
 523 print "\nTEST: wait for factory 3.\n" if ($verbose);
 524 $status_replica = $REP3->WaitKill ($ft_replica3->ProcessStopWaitInterval() + 15);
 525 if ($status_replica != 0) {
 526     print STDERR "TEST ERROR: replica returned $status_replica\n";
 527     exit_and_kill(1);
 528 }
 529
 530 print "\nTEST: shutting down the replication manager.\n" if ($verbose);
 531 $status_controller = $RMC->SpawnWaitKill ($rp_manager_controller->ProcessStartWaitInterval() + 285);
 532 if ($status_controller != 0) {
 533     print STDERR "TEST ERROR: replication manager controller returned $status_controller\n";
 534     $status = 1;
 535 }
 536
 537 print "\nTEST: wait for ReplicationManager.\n" if ($verbose);
 538 $status_repmgr = $RM->WaitKill ($rp_manager->ProcessStopWaitInterval() + 15);
 539 if ($status_repmgr != 0) {
 540     print STDERR "TEST ERROR: ReplicationManager returned $status_repmgr\n";
 541     $status = 1;
 542 }
 543
 544 print "\nTEST: releasing scratch files.\n" if ($verbose);
 545
 546 # Files for rp_manager
 547 $rp_manager->DeleteFile ($rm_ior);
 548
 549 # Files for ft_replica1
 550 $ft_replica1->DeleteFile ($factory1_ior);
 551 $ft_replica1->DeleteFile ($client_data);
 552
 553 # Files for ft_replica2
 554 $ft_replica2->DeleteFile ($factory2_ior);
 555 $ft_replica2->DeleteFile ($client_data);
 556
 557 # Files for ft_replica3
 558 $ft_replica3->DeleteFile ($factory3_ior);
 559 $ft_replica3->DeleteFile ($client_data);
 560
 561 # Files for ft_creator
 562 $ft_creator->DeleteFile ($replica1_ior);
 563 $ft_creator->DeleteFile ($replica2_ior);
 564 $ft_creator->DeleteFile ($replica3_ior);
 565 $ft_creator->DeleteFile ($replica4_ior);
 566 $ft_creator->DeleteFile ($replica5_ior);
 567 $ft_creator->DeleteFile ($replica6_ior);
 568
 569 $ft_creator->DeleteFile ($replica1_iogr);
 570 $ft_creator->DeleteFile ($replica2_iogr);
 571 $ft_creator->DeleteFile ($replica3_iogr);
 572
 573 # Files for client1
 574 $client1->DeleteFile ($replica1_ior);
 575 $client1->DeleteFile ($replica2_ior);
 576 $client1->DeleteFile ($replica1_iogr);
 577
 578 # Files for client2
 579 $client2->DeleteFile ($replica3_ior);
 580 $client2->DeleteFile ($replica4_ior);
 581 $client2->DeleteFile ($replica2_iogr);
 582
 583 # Files for client3
 584 $client3->DeleteFile ($replica5_ior);
 585 $client3->DeleteFile ($replica6_ior);
 586 $client3->DeleteFile ($replica3_iogr);
 587
 588 exit $status;