TAO/orbsvcs/tests/FT_App/run_test_demo.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2      & eval 'exec perl -S $0 $argv:q'
   3     if 0;
   4
   5 # -*- perl -*-
   6
   7 # Purpose:
   8 #   Integration test for all FT services.
   9 #
  10 # Command line options:
  11 #   --debug_build  use exes from this directory
  12 #        if not specified use exes from ./release
  13 #   --no_simulate
  14 #        use real IOGR-based recovery.
  15 #   -v  display test progress messages (repeating option increases verbosity
  16 # Process being tested:
  17 #   FT_ReplicationManager
  18 #   Fault_Detector
  19 #   Fault_Notifier
  20 #
  21 # Processes used in test:
  22 #       FT_Replica * 3
  23 #           implements GenericFactory interface to create TestReplicas
  24 #           TestReplica implements TestReplica interface.
  25 #           TestReplica implements PullMonitorable interface.
  26 #       FT_Client
  27 #           client for TestReplica interface.
  28 #           client for PullMonitorable.
  29 #       Object Group Creator
  30 #           Creates groups of objects.
  31 #
  32 # Test Scenario
  33 #
  34 #   1)        Start the ReplicationManager (RM),
  35 #   2)        Start the Fault Notification Server(FN)
  36 #   2.1)      FN registers with RM.
  37 #   2.2)      RM registers as consumer with FN
  38 #   3)        Start FaultDetectorFactory at location shire (FD@shire)
  39 #   3.1)      FD@shire registers with RM
  40 #   4)        Start FaultDetectorFactory at location bree (FD@bree)
  41 #   4.1)      FD@bree registers with RM
  42 #   5)        Start Replica Factory at location shire (RF@shire) that can create hobbits
  43 #   5.1)      RF@shire registers with RM to create hobbit@shire
  44 #   6)        Start Replica Factory at location bree (RF@bree) that can create hobbits and elves.
  45 #   6.1)      RF@bree registers with RM to create hobbit@bree
  46 #   6.1)      RF@bree registers with RM to create elf@bree
  47 #   6)        Start Replica Factory at location rivendell (RF@rivendell) that can create elves.
  48 #   6.1)      RF@bree registers with RM to create elf@rivendell
  49 #   7)        Start ObjectGroupCreator (OGC)
  50 #   7.1)      OGC calls RM to create group of hobbits (IOGR1)
  51 #   7.1.1)    OGC calls RF@shire to create hobbit@shire[1]
  52 #   7.1.1.1)  OGC calls FD@shire to create FaultDetector for hobbit@shire[1]
  53 #   7.1.1.2)  OGC adds hobbit@shire[1] to IOGR1.
  54 #   7.1.2)    OGC calls RF@bree to craate hobbit@bree[1]
  55 #   7.1.2.1)  OGC calls FD@bree to create FaultDetector for hobbit@bree[1]
  56 #   7.1.2.2)  OGC adds hobbit@bree[1] to IOGR1.
  57 #   7.2)      OGC calls RM to create group of elves (IOGR2)
  58 #   7.2.1)    OGC calls RF@bree to create elf@bree[2]
  59 #   7.2.1.1)  OGC calls FD@bree to create FaultDetector for elf@bree[2]
  60 #   7.2.1.2)  OGC adds elf@bree[2] to IOGR2.
  61 #   7.2.2)    OGC calls RF@rivendell to create elf@rivendell[1]
  62 #   7.2.2.1)  OGC calls FD@shire to create FaultDetector for elf@rivendell[1]
  63 #   7.2.2.2)  OGC adds elf@rivendell[1] to IOGR2.
  64 #   7.3)      OGC calls RM to create group of hobbits (IOGR3)
  65 #   7.3.1)    OGC calls RF@shire to create hobbit@shire[2]
  66 #   7.3.1.1)  OGC calls FD@shire to create FaultDetector for hobbit@shire[2]
  67 #   7.3.1.2)  OGC adds hobbit@shire[2] to IOGR2.
  68 #   7.3.2)    OGC calls RF@bree to craate hobbit@bree[3]
  69 #   7.3.2.1)  OGC calls FD@bree to create FaultDetector for hobbit@bree[3]
  70 #   7.3.2.2)  OGC adds hobbit@bree[3] to IOGR3.
  71 #   8)        Start client1 with IOGR1
  72 #   8.1)      Hobbit@shire[1] fails. FD sends notification FN.
  73 #   8.1.1)    FN sends notification to RM.
  74 #   8.1.2)    RM removes hobbit@shire[1] from IOGR1.
  75 #   8.2)      Client1 terminates itself and hobbit@bree[1].
  76 #   9)        Start Cient2 with IOGR2.
  77 #   9.1)      Repeat 8.1 & 8.2 using IOGR3.
  78 #   10)       Start Cient3 with IOGR3.
  79 #   10.1)     Repeat 8.1 & 8.2 using IOGR3.
  80 #   11)       System manages to shut itself down.
  81 use lib "$ENV{ACE_ROOT}/bin";
  82 use PerlACE::TestTarget;
  83
  84 ########################
  85 #command line options
  86 #set defaults:
  87
  88 my($status) = 0;
  89 my($verbose) = 0;         # 1: report perl actions before executing them
  90 my($debug_builds) = ($^O eq 'MSWin32' ? 0 : 1);    # 0: use exes from Release directories
  91 my($simulated) = 1;       # 1: use "client simulated" fault tolerance
  92
  93 # List of the proccess which must be killed before exit with error
  94 @kill_list = ();
  95
  96 sub add_to_kills{
  97     $goal = shift;
  98     @kill_list = (@kill_list, $goal);
  99 }
 100 sub exit_and_kill{
 101     $status = shift;
 102     foreach $goal (@kill_list){
 103         $goal->Kill (); $goal->TimedWait (1);
 104     }
 105     exit $status;
 106 }
 107
 108 foreach $i (@ARGV) {
 109     if ($i eq "--debug_build"){
 110         $debug_builds = 1;
 111     }
 112     elsif ($i eq "--no_simulate"){  # reverse this once we have FT ORB support
 113         $simulated = 0;
 114     }
 115     elsif ($i eq "-v"){
 116         $verbose += 1;
 117     }
 118 }
 119
 120 my($build_directory) = "/Release";
 121 if ( $debug_builds ) {
 122     $build_directory = "";
 123 }
 124
 125 if ( $verbose > 1) {
 126     print "verbose: $verbose\n";
 127     print "debug_builds: $debug_builds -> $build_directory\n";
 128     print "simulated: $simulated\n";
 129 }
 130
 131 my $rp_manager = PerlACE::TestTarget::create_target (1) || die "Create target 1 failed\n";
 132 my $rp_manager_controller = PerlACE::TestTarget::create_target (2) || die "Create target 2 failed\n";
 133 my $fault_detector1 = PerlACE::TestTarget::create_target (3) || die "Create target 3 failed\n";
 134 my $fault_detector2 = PerlACE::TestTarget::create_target (4) || die "Create target 4 failed\n";
 135 my $fault_notifier = PerlACE::TestTarget::create_target (5) || die "Create target 5 failed\n";
 136 my $ft_replica1 = PerlACE::TestTarget::create_target (6) || die "Create target 6 failed\n";
 137 my $ft_replica2 = PerlACE::TestTarget::create_target (7) || die "Create target 7 failed\n";
 138 my $ft_replica3 = PerlACE::TestTarget::create_target (8) || die "Create target 8 failed\n";
 139 my $ft_creator = PerlACE::TestTarget::create_target (9) || die "Create target 9 failed\n";
 140 my $client1 = PerlACE::TestTarget::create_target (10) || die "Create target 10 failed\n";
 141 my $client2 = PerlACE::TestTarget::create_target (11) || die "Create target 11 failed\n";
 142 my $client3 = PerlACE::TestTarget::create_target (12) || die "Create target 12 failed\n";
 143
 144 my($role1) = "hobbit";
 145 my($role2) = "elf";
 146
 147 my($location1) = "shire";
 148 my($location2) = "bree";
 149 my($location3) = "rivendell";
 150
 151 #define temp files
 152 my($rm_ior) = "rm.ior";
 153 my($factory1_ior) = "factory1.ior";
 154 my($factory2_ior) = "factory2.ior";
 155 my($factory3_ior) = "factory3.ior";
 156 my($detector1_ior) = "detector1.ior";
 157 my($detector2_ior) = "detector2.ior";
 158 my($notifier_ior) = "notifier.ior";
 159
 160 my($replica1_ior) = "${role1}_${location1}_0.ior";
 161 my($replica2_ior) = "${role1}_${location2}_0.ior";
 162 my($replica3_ior) = "${role2}_${location2}_1.ior";
 163 my($replica4_ior) = "${role2}_${location3}_0.ior";
 164 my($replica5_ior) = "${role1}_${location1}_1.ior";
 165 my($replica6_ior) = "${role1}_${location2}_2.ior";
 166
 167 my($replica1_iogr) = "${role1}_0.iogr";
 168 my($replica2_iogr) = "${role2}_1.iogr";
 169 my($replica3_iogr) = "${role1}_2.iogr";
 170
 171 my($client_data) = "persistent.dat";
 172
 173 # Files for rp_manager
 174 my $rp_manager_ior = $rp_manager->LocalFile ($rm_ior);
 175 $rp_manager->DeleteFile ($rm_ior);
 176
 177 # Files for fault_detector1
 178 my $fault_detector1_ior = $fault_detector1->LocalFile ($detector1_ior);
 179 $fault_detector1->DeleteFile ($detector1_ior);
 180
 181 # Files for fault_detector2
 182 my $fault_detector2_ior = $fault_detector2->LocalFile ($detector2_ior);
 183 $fault_detector2->DeleteFile ($detector2_ior);
 184
 185 # Files for fault_notifier
 186 my $fault_notifier_ior = $fault_notifier->LocalFile ($notifier_ior);
 187 $fault_notifier->DeleteFile ($notifier_ior);
 188
 189 # Files for ft_replica1
 190 my $ft_replica1_factory1_ior = $ft_replica1->LocalFile ($factory1_ior);
 191 my $ft_replica1_client_data = $ft_replica1->LocalFile ($client_data);
 192
 193 $ft_replica1->DeleteFile ($factory1_ior);
 194 $ft_replica1->DeleteFile ($client_data);
 195
 196 # Files for ft_replica2
 197 my $ft_replica2_factory2_ior = $ft_replica2->LocalFile ($factory2_ior);
 198 my $ft_replica2_client_data = $ft_replica2->LocalFile ($client_data);
 199
 200 $ft_replica2->DeleteFile ($factory2_ior);
 201 $ft_replica2->DeleteFile ($client_data);
 202
 203 # Files for ft_replica3
 204 my $ft_replica3_factory3_ior = $ft_replica3->LocalFile ($factory3_ior);
 205 my $ft_replica3_client_data = $ft_replica3->LocalFile ($client_data);
 206
 207 $ft_replica3->DeleteFile ($factory3_ior);
 208 $ft_replica3->DeleteFile ($client_data);
 209
 210 # Files for ft_creator
 211 my $creator_replica1_ior = $ft_creator->LocalFile ($replica1_ior);
 212 my $creator_replica2_ior = $ft_creator->LocalFile ($replica2_ior);
 213 my $creator_replica3_ior = $ft_creator->LocalFile ($replica3_ior);
 214 my $creator_replica4_ior = $ft_creator->LocalFile ($replica4_ior);
 215 my $creator_replica5_ior = $ft_creator->LocalFile ($replica5_ior);
 216 my $creator_replica6_ior = $ft_creator->LocalFile ($replica6_ior);
 217
 218 my $creator_replica1_iogr = $ft_creator->LocalFile ($replica1_iogr);
 219 my $creator_replica2_iogr = $ft_creator->LocalFile ($replica2_iogr);
 220 my $creator_replica3_iogr = $ft_creator->LocalFile ($replica3_iogr);
 221
 222 $ft_creator->DeleteFile ($replica1_ior);
 223 $ft_creator->DeleteFile ($replica2_ior);
 224 $ft_creator->DeleteFile ($replica3_ior);
 225 $ft_creator->DeleteFile ($replica4_ior);
 226 $ft_creator->DeleteFile ($replica5_ior);
 227 $ft_creator->DeleteFile ($replica6_ior);
 228
 229 $ft_creator->DeleteFile ($replica1_iogr);
 230 $ft_creator->DeleteFile ($replica2_iogr);
 231 $ft_creator->DeleteFile ($replica3_iogr);
 232
 233 # Files for client1
 234 my $client1_replica1_ior = $client1->LocalFile ($replica1_ior);
 235 my $client1_replica2_ior = $client1->LocalFile ($replica2_ior);
 236 my $client1_replica1_iogr = $client1->LocalFile ($replica1_iogr);
 237
 238 $client1->DeleteFile ($replica1_ior);
 239 $client1->DeleteFile ($replica2_ior);
 240 $client1->DeleteFile ($replica1_iogr);
 241
 242 # Files for client2
 243 my $client2_replica3_ior = $client2->LocalFile ($replica3_ior);
 244 my $client2_replica4_ior = $client2->LocalFile ($replica4_ior);
 245 my $client2_replica2_iogr = $client2->LocalFile ($replica2_iogr);
 246
 247 $client2->DeleteFile ($replica3_ior);
 248 $client2->DeleteFile ($replica4_ior);
 249 $client2->DeleteFile ($replica2_iogr);
 250
 251 # Files for client3
 252 my $client3_replica5_ior = $client3->LocalFile ($replica5_ior);
 253 my $client3_replica6_ior = $client3->LocalFile ($replica6_ior);
 254 my $client3_replica3_iogr = $client3->LocalFile ($replica3_iogr);
 255
 256 $client3->DeleteFile ($replica5_ior);
 257 $client3->DeleteFile ($replica6_ior);
 258 $client3->DeleteFile ($replica3_iogr);
 259
 260 $hostname = $rp_manager->HostName ();
 261 $port = $rp_manager->RandomPort ();
 262
 263 $RM = $rp_manager->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager" .
 264                                   "$build_directory/tao_ft_replicationmanager",
 265                                   "-o $rp_manager_ior " .
 266                                   "-ORBEndpoint iiop://$hostname:$port");
 267
 268 $RMC = $rp_manager_controller->CreateProcess (".$build_directory/replmgr_controller" ,
 269                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 270                                  "-x");
 271
 272 $NOT = $fault_notifier->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier" .
 273                                        "$build_directory/tao_fault_notifier" ,
 274                                         "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 275                                         "-o $fault_notifier_ior -q");
 276
 277 $DET1 = $fault_detector1->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector" .
 278                                        "$build_directory/tao_fault_detector" ,
 279                                        "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 280                                        "-o $fault_detector1_ior " .
 281                                        "-l $location1 -q");
 282
 283 $DET2 = $fault_detector2->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector" .
 284                                        "$build_directory/tao_fault_detector" ,
 285                                        "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 286                                        "-o $fault_detector1_ior " .
 287                                        "-l $location2 -q");
 288
 289 $REP1 = $ft_replica1->CreateProcess (".$build_directory/ft_replica" ,
 290                                      "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 291                                      "-o $ft_replica1_factory1_ior ".
 292                                      "-l $location1 -i $role1 -q -p $ft_replica1_client_data");
 293
 294 $REP2 = $ft_replica2->CreateProcess (".$build_directory/ft_replica" ,
 295                                      "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 296                                      "-o $ft_replica2_factory2_ior ".
 297                                      "-l $location2 -i $role1 -i $role2 -q -p $ft_replica2_client_data");
 298
 299 $REP3 = $ft_replica3->CreateProcess (".$build_directory/ft_replica" ,
 300                                      "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 301                                      "-o $ft_replica3_factory3_ior ".
 302                                      "-l $location3 -i $role2 -q -p $ft_replica3_client_data");
 303
 304 my($OGC);
 305 my($CL1);
 306 my($CL2);
 307 my($CL3);
 308 if ($simulated) {
 309     print "\nTEST: Preparing Client Mediated Fault Tolerance test.\n" if ($verbose);
 310     $OGC = $ft_creator->CreateProcess (".$build_directory/ft_create" ,
 311                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 312                                  "-n -r $role1 -r $role2 -r $role1 -u $role3 -i");
 313
 314     $CL1 = $client1->CreateProcess (".$build_directory/ft_client" ,
 315                                  "-f file://$client1_replica1_ior " .
 316                                  "-f file://$client1_replica2_ior " .
 317                                  "-c testscript");
 318
 319     $CL2 = $client2->CreateProcess (".$build_directory/ft_client" ,
 320                                  "-f file://$client2_replica3_ior " .
 321                                  "-f file://$client2_replica4_ior " .
 322                                  "-c testscript");
 323
 324     $CL3 = $client3->CreateProcess (".$build_directory/ft_client" ,
 325                                  "-f file://$client3_replica5_ior " .
 326                                  "-f file://$client3_replica6_ior " .
 327                                  "-c testscript");
 328
 329
 330
 331 }else{
 332     print "\nTEST: Preparing IOGR based test.\n" if ($verbose);
 333     $OGC = $ft_creator->CreateProcess (".$build_directory/ft_create" ,
 334                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 335                                  "-n -r $role1 -r $role2 -r $role1 -u $role3 -g");
 336
 337     $CL1 = $client1->CreateProcess (".$build_directory/ft_client" ,
 338                                  "-f file://$client1_replica1_iogr " .
 339                                  "-c testscript");
 340
 341     $CL2 = $client2->CreateProcess (".$build_directory/ft_client" ,
 342                                  "-f file://$client2_replica2_iogr " .
 343                                  "-c testscript");
 344
 345     $CL3 = $client3->CreateProcess (".$build_directory/ft_client" ,
 346                                  "-f file://$client3_replica3_iogr " .
 347                                  "-c testscript");
 348 }
 349
 350
 351 #######################
 352 # ReplicationManager
 353
 354 print "\nTEST: starting ReplicationManager " . $RM->CommandLine . "\n" if ($verbose);
 355 $RM->Spawn ();
 356
 357 add_to_kills ($RM);
 358
 359 print "TEST: waiting for registry's IOR\n" if ($verbose);
 360
 361 if ($rp_manager->WaitForFileTimed ($rm_ior,
 362                                $rp_manager->ProcessStartWaitInterval()) == -1) {
 363     print STDERR "ERROR: cannot find file <$rp_manager_ior>\n";
 364     exit_and_kill (1);
 365 }
 366
 367 #######################
 368 # Start FaultNotifier
 369 print "\nTEST: starting notifier " . $NOT->CommandLine . "\n" if ($verbose);
 370 $NOT->Spawn ();
 371
 372 add_to_kills ($NOT);
 373
 374 print "TEST: waiting for notifier's IOR\n" if ($verbose);
 375 if ($fault_notifier->WaitForFileTimed ($notifier_ior,
 376                                $fault_notifier->ProcessStartWaitInterval()) == -1) {
 377     print STDERR "ERROR: cannot find file <$fault_notifier_ior>\n";
 378     exit_and_kill (1);
 379 }
 380
 381 ##############################
 382 # Start FaultDetectorFactory 1
 383 print "\nTEST: Starting FaultDetectorFactory at $location1 " . $DET1->CommandLine . "\n" if ($verbose);
 384 $DET1->Spawn ();
 385
 386 add_to_kills ($DET1);
 387
 388 print "TEST: waiting for FaultDetector's IOR\n" if ($verbose);
 389 if ($fault_detector1->WaitForFileTimed ($detector1_ior,
 390                                $fault_detector1->ProcessStartWaitInterval()) == -1) {
 391     print STDERR "ERROR: cannot find file <$fault_detector1_ior>\n";
 392     exit_and_kill (1);
 393 }
 394
 395 ##############################
 396 # Start FaultDetectorFactory 2
 397 print "\nTEST: Starting FaultDetectorFactory at $location2 " . $DET2->CommandLine . "\n" if ($verbose);
 398 $DET2->Spawn ();
 399
 400 add_to_kills ($DET2);
 401
 402 print "TEST: waiting for FaultDetector's IOR\n" if ($verbose);
 403 if ($fault_detector2->WaitForFileTimed ($detector2_ior,
 404                                $fault_detector2->ProcessStartWaitInterval()) == -1) {
 405     print STDERR "ERROR: cannot find file <$fault_detector1_ior>\n";
 406     exit_and_kill (1);
 407 }
 408
 409 #################
 410 # Start Factories
 411
 412 print "\nTEST: starting factory 1 " . $REP1->CommandLine . "\n" if ($verbose);
 413 $REP1->Spawn ();
 414
 415 add_to_kills ($REP1);
 416
 417 print "TEST: waiting for factory 1's IOR\n" if ($verbose);
 418 if ($ft_replica1->WaitForFileTimed ($factory1_ior,
 419                                $ft_replica1->ProcessStartWaitInterval()) == -1) {
 420     print STDERR "ERROR: cannot find file <$ft_replica1_factory1_ior>\n";
 421     exit_and_kill (1);
 422 }
 423
 424 print "\nTEST: starting factory 2 " . $REP2->CommandLine . "\n" if ($verbose);
 425 $REP2->Spawn ();
 426
 427 add_to_kills ($REP2);
 428
 429 print "TEST: waiting for factory 2's IOR\n" if ($verbose);
 430 if ($ft_replica2->WaitForFileTimed ($factory2_ior,
 431                                $ft_replica2->ProcessStartWaitInterval()) == -1) {
 432     print STDERR "ERROR: cannot find file <$ft_replica2_factory2_ior>\n";
 433     exit_and_kill (1);
 434 }
 435
 436 print "\nTEST: starting factory 3 " . $REP3->CommandLine . "\n" if ($verbose);
 437 $REP3->Spawn ();
 438
 439 add_to_kills ($REP3);
 440
 441 print "TEST: waiting for factory 3's IOR\n" if ($verbose);
 442 if ($ft_replica3->WaitForFileTimed ($factory3_ior,
 443                                $ft_replica3->ProcessStartWaitInterval()) == -1) {
 444     print STDERR "ERROR: cannot find file <$ft_replica3_factory3_ior>\n";
 445     exit_and_kill (1);
 446 }
 447 ######################
 448 # Create object groups
 449 print "\nTEST: Starting object group creator " . $OGC->CommandLine . "\n" if ($verbose);
 450 $OGC->Spawn ();
 451
 452 print "\nTEST: wait for object group creator.\n" if ($verbose);
 453
 454 $status_creator = $OGC->WaitKill ($ft_creator->ProcessStartWaitInterval());
 455
 456 if ($status_creator != 0) {
 457     print STDERR "TEST ERROR: Object Group Creator returned $status_creator\n";
 458     exit_and_kill (1);
 459 }
 460 # Get/Put files:
 461 # replica1_ior: client1
 462 if ($ft_creator->GetFile ($replica1_ior) == -1) {
 463     print STDERR "ERROR: cannot retrieve file <$creator_replica1_ior\n";
 464     exit_and_kill (1);
 465 }
 466 if ($client1->PutFile ($replica1_ior) == -1) {
 467     print STDERR "ERROR: cannot set file <$client1_replica1_ior>\n";
 468     exit_and_kill (1);
 469 }
 470 # replica2_ior: client1
 471 if ($ft_creator->GetFile ($replica2_ior) == -1) {
 472     print STDERR "ERROR: cannot retrieve file <$creator_replica2_ior\n";
 473     exit_and_kill (1);
 474 }
 475 if ($client1->PutFile ($replica2_ior) == -1) {
 476     print STDERR "ERROR: cannot set file <$client1_replica2_ior>\n";
 477     exit_and_kill (1);
 478 }
 479 # replica3_ior: client2
 480 if ($ft_creator->GetFile ($replica3_ior) == -1) {
 481     print STDERR "ERROR: cannot retrieve file <$creator_replica3_ior\n";
 482     exit_and_kill (1);
 483 }
 484 if ($client2->PutFile ($replica3_ior) == -1) {
 485     print STDERR "ERROR: cannot set file <$client2_replica3_ior>\n";
 486     exit_and_kill (1);
 487 }
 488 # replica4_ior: client2
 489 if ($ft_creator->GetFile ($replica4_ior) == -1) {
 490     print STDERR "ERROR: cannot retrieve file <$creator_replica4_ior\n";
 491     exit_and_kill (1);
 492 }
 493 if ($client2->PutFile ($replica4_ior) == -1) {
 494     print STDERR "ERROR: cannot set file <$client2_replica4_ior>\n";
 495     exit_and_kill (1);
 496 }
 497 # replica5_ior: client3
 498 if ($ft_creator->GetFile ($replica5_ior) == -1) {
 499     print STDERR "ERROR: cannot retrieve file <$creator_replica5_ior\n";
 500     exit_and_kill (1);
 501 }
 502 if ($client3->PutFile ($replica5_ior) == -1) {
 503     print STDERR "ERROR: cannot set file <$client3_replica5_ior>\n";
 504     exit_and_kill (1);
 505 }
 506 # replica6_ior: client3
 507 if ($ft_creator->GetFile ($replica6_ior) == -1) {
 508     print STDERR "ERROR: cannot retrieve file <$creator_replica6_ior\n";
 509     exit_and_kill (1);
 510 }
 511 if ($client3->PutFile ($replica6_ior) == -1) {
 512     print STDERR "ERROR: cannot set file <$client3_replica6_ior>\n";
 513     exit_and_kill (1);
 514 }
 515
 516 #############
 517 # Run clients
 518
 519 print "\nTEST: Starting client using first group of hobbits " . $CL1->CommandLine . "\n" if ($verbose);
 520 $status_client = $CL1->SpawnWaitKill ($client1->ProcessStartWaitInterval() + 45);
 521 if ($status_client != 0) {
 522     print STDERR "TEST ERROR: client returned $status_client\n";
 523     exit_and_kill(1);
 524 }
 525
 526 print "\nTEST: Starting client using group of elves " . $CL2->CommandLine . "\n" if ($verbose);
 527 $status_client = $CL2->SpawnWaitKill ($client2->ProcessStartWaitInterval() + 45);
 528 if ($status_client != 0) {
 529     print STDERR "TEST ERROR: client returned $status_client\n";
 530     exit_and_kill(1);
 531 }
 532
 533 print "\nTEST: Starting client using second group of hobbits " . $CL3->CommandLine . "\n" if ($verbose);
 534 $status_client = $CL3->SpawnWaitKill ($client3->ProcessStartWaitInterval() + 45);
 535 if ($status_client != 0) {
 536     print STDERR "TEST ERROR: client returned $status_client\n";
 537     exit_and_kill(1);
 538 }
 539
 540 ######################
 541 # Clean house and exit
 542
 543 print "\nTEST: wait for factory 1.\n" if ($verbose);
 544 $status_replica = $REP1->WaitKill ($ft_replica1->ProcessStopWaitInterval() + 15);
 545 if ($status_replica != 0) {
 546     print STDERR "TEST ERROR: replica returned $status_replica\n";
 547     $status = 1;
 548 }
 549 print "\nTEST: wait for factory 2.\n" if ($verbose);
 550 $status_replica = $REP2->WaitKill ($ft_replica2->ProcessStopWaitInterval() + 15);
 551 if ($status_replica != 0) {
 552     print STDERR "TEST ERROR: replica returned $status_replica\n";
 553     $status = 1;
 554 }
 555
 556 print "\nTEST: wait for factory 3.\n" if ($verbose);
 557 $status_replica = $REP3->WaitKill ($ft_replica3->ProcessStopWaitInterval() + 15);
 558 if ($status_replica != 0) {
 559     print STDERR "TEST ERROR: replica returned $status_replica\n";
 560     $status = 1;
 561 }
 562 print "\nTEST: wait for FaultDetectorFactory 1.\n" if ($verbose);
 563 $status_detector = $DET1->WaitKill ($fault_detector1->ProcessStopWaitInterval() + 15);
 564 if ($status_detector != 0) {
 565     print STDERR "TEST ERROR: FaultDetectorFactory returned $status_detector\n";
 566     $status = 1;
 567 }
 568 print "\nTEST: wait for FaultDetectorFactory 2.\n" if ($verbose);
 569 $status_detector = $DET2->WaitKill ($fault_detector2->ProcessStopWaitInterval() + 15);
 570 if ($status_detector != 0) {
 571     print STDERR "TEST ERROR: FaultDetectorFactory returned $status_detector\n";
 572     $status = 1;
 573 }
 574
 575 print "\nTEST: shutting down the replication manager.\n" if ($verbose);
 576 $status_controller = $RMC->SpawnWaitKill ($rp_manager_controller->ProcessStartWaitInterval() + 285);
 577 if ($status_controller != 0) {
 578     print STDERR "TEST ERROR: replication manager controller returned $status_controller\n";
 579     $status = 1;
 580 }
 581 print "\nTEST: wait for ReplicationManager.\n" if ($verbose);
 582 $status_repmgr = $RM->WaitKill ($rp_manager->ProcessStopWaitInterval() + 15);
 583 if ($status_repmgr != 0) {
 584     print STDERR "TEST ERROR: ReplicationManager returned $status_repmgr\n";
 585     $status = 1;
 586 }
 587
 588 print "\nTEST: wait for FaultNotifier.\n" if ($verbose);
 589 $status_not = $NOT->WaitKill ($fault_notifier->ProcessStopWaitInterval());
 590 if ($status_not != 0) {
 591     print STDERR "TEST ERROR: notifier returned $status_not\n";
 592     $status = 1;
 593 }
 594
 595 print "\nTEST: releasing scratch files.\n" if ($verbose);
 596
 597 # Files for rp_manager
 598 $rp_manager->DeleteFile ($rm_ior);
 599
 600 # Files for fault_detector1
 601 $fault_detector1->DeleteFile ($detector1_ior);
 602
 603 # Files for fault_detector2
 604 $fault_detector2->DeleteFile ($detector2_ior);
 605
 606 # Files for fault_notifier
 607 $fault_notifier->DeleteFile ($notifier_ior);
 608
 609 # Files for ft_replica1
 610 $ft_replica1->DeleteFile ($factory1_ior);
 611 $ft_replica1->DeleteFile ($client_data);
 612
 613 # Files for ft_replica2
 614 $ft_replica2->DeleteFile ($factory2_ior);
 615 $ft_replica2->DeleteFile ($client_data);
 616
 617 # Files for ft_replica3
 618 $ft_replica3->DeleteFile ($factory3_ior);
 619 $ft_replica3->DeleteFile ($client_data);
 620
 621 # Files for ft_creator
 622 $ft_creator->DeleteFile ($replica1_ior);
 623 $ft_creator->DeleteFile ($replica2_ior);
 624 $ft_creator->DeleteFile ($replica3_ior);
 625 $ft_creator->DeleteFile ($replica4_ior);
 626 $ft_creator->DeleteFile ($replica5_ior);
 627 $ft_creator->DeleteFile ($replica6_ior);
 628
 629 $ft_creator->DeleteFile ($replica1_iogr);
 630 $ft_creator->DeleteFile ($replica2_iogr);
 631 $ft_creator->DeleteFile ($replica3_iogr);
 632
 633 # Files for client1
 634 $client1->DeleteFile ($replica1_ior);
 635 $client1->DeleteFile ($replica2_ior);
 636 $client1->DeleteFile ($replica1_iogr);
 637
 638 # Files for client2
 639 $client2->DeleteFile ($replica3_ior);
 640 $client2->DeleteFile ($replica4_ior);
 641 $client2->DeleteFile ($replica2_iogr);
 642
 643 # Files for client3
 644 $client3->DeleteFile ($replica5_ior);
 645 $client3->DeleteFile ($replica6_ior);
 646 $client3->DeleteFile ($replica3_iogr);
 647
 648 exit $status;