TAO/orbsvcs/tests/FT_App/run_test_registry.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2      & eval 'exec perl -S $0 $argv:q'
   3     if 0;
   4
   5 # -*- perl -*-
   6
   7 # Purpose:
   8 #   To test the FactoryRegistry either as a stand-alone application or as part
   9 #   of the replication manager
  10 #
  11 # Command line options:
  12 #   --debug_build  use exes from this directory
  13 #        if not specified use exes from ./release
  14 #   -r use ReplicationManager rather than stand-alone factory
  15 #   --no_simulate
  16 #        use real IOGR-based recovery.
  17 #   -v  display test progress messages (repeating option increases verbosity
  18 #
  19 # Process being tested:
  20 #       FT_Registry
  21 #           implements PortableGroup::FactoryRegistry interface.
  22 #  or
  23 #       FT_ReplicationManager (if -r option is used)
  24 #
  25 # Processes used in test:
  26 #       FT_Replica * 3
  27 #           implements GenericFactory interface to create TestReplicas
  28 #           TestReplica implements TestReplica interface.
  29 #           TestReplica implements PullMonitorable interface.
  30 #       FT_Client
  31 #           client for TestReplica interface.
  32 #           client for PullMonitorable.
  33 #       Object Group Creator
  34 #           Creates groups of objects.
  35 #
  36 # Test Scenario (***Test: marks behavior being tested):
  37 #
  38 #   First the test starts a factory registry,
  39 #   Then starts three factories.  Each factory exists at a separate location.
  40 #   The locations are named shire, bree, and rivendell.  In a "real" system
  41 #   these locations would be on separate computers.
  42 #
  43 #   The factory at the shire location knows how to create hobbits.
  44 #   The factory at the bree location knows how to create hobbits, elves, and humans.
  45 #   The factory at rivendell can create elves.
  46 #   Hobbits, elves, and humans are actually TestReplica objects.  A creation parameter
  47 #   included as part of the registration information specifies which species they are.
  48 #
  49 #   ***Test: The factories register themselves with the factory registry.  Registration
  50 #   information includes the location, the type of object created, a object reference
  51 #   to the factory and set of parameters to be passed to the factory's create_object
  52 #   method.
  53 #
  54 #   An object group creator is started and asked to create three object groups:
  55 #   a group of hobbits, a group of elves; and another group of hobbits.
  56 #
  57 #   ***Test: the object group creator asks the registry for the set of factories
  58 #   that can create the type of object needed, then uses the create_object method
  59 #   for each factory to create the actual object.
  60 #
  61 #   [temporary until IOGR's are working:  The object group creator writes the
  62 #   IOR's of the create objects to files -- using a file naming convention to
  63 #   distinguish members of the group.  It will be enhanced to assemble these
  64 #   IORs into an IOGR and either write the IOGR to a file or register it with
  65 #   the Naming Service.]
  66 #
  67 #   The object group creator is also told to unregister all factories that create humans.
  68 #   ***Test: It does so using the unregister_factory_by_type method.
  69 #
  70 #   Three clients are started, one at a time.  Each client is given a reference
  71 #   to an object group
  72 #
  73 #   [temporary until IOGRs and transparent reinvocaton work:  each client is
  74 #   given references to the members of the group and manages its own recovery
  75 #   (client mediated fault tolerance)]
  76 #
  77 #   Each client sends a few requests to first member of the object group.  Eventually
  78 #   this member fails and the requests are then rerouted to the second (and last)
  79 #   member of the group.
  80 #
  81 #   When a clients job is done, it exits and asks the remaining group member to
  82 #   exit, too.
  83 #
  84 #   The factories are run with the quit-on-idle option so when the last object
  85 #   created at that location goes away, the factory exits.
  86 #
  87 #   ***Test: As it exits the factory unregisters itself with the factory registry.
  88 #   ***Test: A command line option determines whether it uses a single
  89 #   unregister_factory_by_location call, or separate unregister_factory calles for
  90 #   each type of object created.  In this test, the shire location uses unregister_factory,
  91 #   and bree and rivendell use unregister_factory_by_location.
  92 #
  93 #   The factory registry is also run with the quit-on-idle option so when the last
  94 #   factory unregisters itself, the factory registry shuts down to end the test.
  95 use lib "$ENV{ACE_ROOT}/bin";
  96 use PerlACE::TestTarget;
  97
  98 ########################
  99 #command line options
 100 #set defaults:
 101 my($status) = 0;
 102 my($verbose) = 0;         # 1: report perl actions before executing them
 103 my($debug_builds) = 0;    # 0: use exes from Release directories
 104 my($simulated) = 1;       # 1: use "client simulated" fault tolerance
 105
 106 foreach $i (@ARGV) {
 107     if ($i eq "--debug_build"){
 108         $debug_builds = 1;
 109     }
 110     elsif ($i eq "-r"){ # use RegistrationManager
 111         $use_rm = 1
 112     }
 113     elsif ($i eq "--no_simulate"){  # reverse this once we have FT ORB support
 114         $simulated = 0;
 115     }
 116     elsif ($i eq "-v"){
 117         $verbose += 1;
 118     }
 119     else{
 120         print "unknown option $i.  Expecting: --debug_build, -r --no_simulate, -v\n";
 121         exit(-1);
 122     }
 123 }
 124
 125 my($build_directory) = "/Release";
 126 if ( $debug_builds ) {
 127     $build_directory = "";
 128 }
 129
 130 if ( $verbose > 1) {
 131     print "verbose: $verbose\n";
 132     print "debug_builds: $debug_builds -> $build_directory\n";
 133     print "simulated: $simulated\n";
 134 }
 135
 136 # List of the proccess which must be killed before exit with error
 137 @kill_list = ();
 138
 139 sub add_to_kills{
 140     $goal = shift;
 141     @kill_list = (@kill_list, $goal);
 142 }
 143 sub exit_and_kill{
 144     $status = shift;
 145     foreach $goal (@kill_list){
 146         $goal->Kill (); $goal->TimedWait (1);
 147     }
 148     exit $status;
 149 }
 150
 151 my $rp_manager = PerlACE::TestTarget::create_target (1) || die "Create target 1 failed\n";
 152 my $rp_manager_controller = PerlACE::TestTarget::create_target (2) || die "Create target 2 failed\n";
 153 my $ft_replica1 = PerlACE::TestTarget::create_target (3) || die "Create target 3 failed\n";
 154 my $ft_replica2 = PerlACE::TestTarget::create_target (4) || die "Create target 4 failed\n";
 155 my $ft_replica3 = PerlACE::TestTarget::create_target (5) || die "Create target 5 failed\n";
 156 my $ft_creator = PerlACE::TestTarget::create_target (6) || die "Create target 6 failed\n";
 157 my $client1 = PerlACE::TestTarget::create_target (7) || die "Create target 7 failed\n";
 158 my $client2 = PerlACE::TestTarget::create_target (8) || die "Create target 8 failed\n";
 159 my $client3 = PerlACE::TestTarget::create_target (9) || die "Create target 9 failed\n";
 160 my $ft_registry = PerlACE::TestTarget::create_target (10) || die "Create target 10 failed\n";
 161
 162 my($role1) = "hobbit";
 163 my($role2) = "elf";
 164 my($role3) = "human";
 165
 166 my($location1) = "shire";
 167 my($location2) = "bree";
 168 my($location3) = "rivendell";
 169 my($location4) = "rohan";
 170
 171 my($rm_ior) = "rm.ior";
 172 my($registry_ior) = "registry.ior";
 173
 174 my($factory1_ior) = "factory1.ior";
 175 my($factory2_ior) = "factory2.ior";
 176 my($factory3_ior) = "factory3.ior";
 177
 178 my($replica1_ior) = "${role1}_${location1}_0.ior";
 179 my($replica2_ior) = "${role1}_${location2}_0.ior";
 180 my($replica3_ior) = "${role2}_${location2}_1.ior";
 181 my($replica4_ior) = "${role2}_${location3}_0.ior";
 182 my($replica5_ior) = "${role1}_${location1}_1.ior";
 183 my($replica6_ior) = "${role1}_${location2}_2.ior";
 184
 185 my($replica1_iogr) = "${role1}_0.iogr";
 186 my($replica2_iogr) = "${role2}_1.iogr";
 187 my($replica3_iogr) = "${role1}_2.iogr";
 188
 189 my($client_data) = "persistent.dat";
 190
 191 # Files for rp_manager
 192 my $rp_manager_ior = $rp_manager->LocalFile ($rm_ior);
 193 $rp_manager->DeleteFile ($rm_ior);
 194
 195 # Files for ft_registry
 196 my $ft_registry_registry_ior = $ft_registry->LocalFile ($registry_ior);
 197 $ft_registry->DeleteFile ($registry_ior);
 198
 199 # Files for rp_manager_controller
 200
 201 my ($rp_manager_controller_registry_ior);
 202 if(!$use_rm){
 203     $rp_manager_controller_registry_ior = $rp_manager_controller->LocalFile ($registry_ior);
 204     $rp_manager_controller->DeleteFile ($registry_ior);
 205 }
 206
 207 # Files for ft_replica1
 208 my $ft_replica1_factory1_ior = $ft_replica1->LocalFile ($factory1_ior);
 209 my $ft_replica1_client_data = $ft_replica1->LocalFile ($client_data);
 210
 211 $ft_replica1->DeleteFile ($factory1_ior);
 212 $ft_replica1->DeleteFile ($client_data);
 213
 214 my($ft_replica1_registry_ior);
 215
 216 if(!$use_rm){
 217     $ft_replica1_registry_ior = $ft_replica1->LocalFile ($registry_ior);
 218     $ft_replica1->DeleteFile ($registry_ior);
 219 }
 220
 221 # Files for ft_replica2
 222 my $ft_replica2_factory2_ior = $ft_replica2->LocalFile ($factory2_ior);
 223 my $ft_replica2_client_data = $ft_replica2->LocalFile ($client_data);
 224
 225 $ft_replica2->DeleteFile ($factory2_ior);
 226 $ft_replica2->DeleteFile ($client_data);
 227
 228 my($ft_replica2_registry_ior);
 229 if(!$use_rm){
 230     $ft_replica2_registry_ior = $ft_replica2->LocalFile ($registry_ior);
 231     $ft_replica2->DeleteFile ($registry_ior);
 232 }
 233
 234
 235 # Files for ft_replica3
 236 my $ft_replica3_factory3_ior = $ft_replica3->LocalFile ($factory3_ior);
 237 my $ft_replica3_client_data = $ft_replica3->LocalFile ($client_data);
 238
 239 $ft_replica3->DeleteFile ($factory3_ior);
 240 $ft_replica3->DeleteFile ($client_data);
 241
 242 my($ft_replica3_registry_ior);
 243 if(!$use_rm){
 244     $ft_replica3_registry_ior = $ft_replica3->LocalFile ($registry_ior);
 245     $ft_replica3->DeleteFile ($registry_ior);
 246 }
 247
 248 # Files for ft_creator
 249 my $creator_replica1_ior = $ft_creator->LocalFile ($replica1_ior);
 250 my $creator_replica2_ior = $ft_creator->LocalFile ($replica2_ior);
 251 my $creator_replica3_ior = $ft_creator->LocalFile ($replica3_ior);
 252 my $creator_replica4_ior = $ft_creator->LocalFile ($replica4_ior);
 253 my $creator_replica5_ior = $ft_creator->LocalFile ($replica5_ior);
 254 my $creator_replica6_ior = $ft_creator->LocalFile ($replica6_ior);
 255
 256 my $creator_replica1_iogr = $ft_creator->LocalFile ($replica1_iogr);
 257 my $creator_replica2_iogr = $ft_creator->LocalFile ($replica2_iogr);
 258 my $creator_replica3_iogr = $ft_creator->LocalFile ($replica3_iogr);
 259
 260 my($ft_creator_registry_ior);
 261 if(!$use_rm){
 262     $ft_creator_registry_ior = $ft_creator->LocalFile ($registry_ior);
 263     $ft_creator->DeleteFile ($registry_ior);
 264 }
 265
 266 $ft_creator->DeleteFile ($replica1_ior);
 267 $ft_creator->DeleteFile ($replica2_ior);
 268 $ft_creator->DeleteFile ($replica3_ior);
 269 $ft_creator->DeleteFile ($replica4_ior);
 270 $ft_creator->DeleteFile ($replica5_ior);
 271 $ft_creator->DeleteFile ($replica6_ior);
 272
 273 $ft_creator->DeleteFile ($replica1_iogr);
 274 $ft_creator->DeleteFile ($replica2_iogr);
 275 $ft_creator->DeleteFile ($replica3_iogr);
 276
 277 # Files for client1
 278 my $client1_replica1_ior = $client1->LocalFile ($replica1_ior);
 279 my $client1_replica2_ior = $client1->LocalFile ($replica2_ior);
 280 my $client1_replica1_iogr = $client1->LocalFile ($replica1_iogr);
 281
 282 $client1->DeleteFile ($replica1_ior);
 283 $client1->DeleteFile ($replica2_ior);
 284 $client1->DeleteFile ($replica1_iogr);
 285
 286 # Files for client2
 287 my $client2_replica3_ior = $client2->LocalFile ($replica3_ior);
 288 my $client2_replica4_ior = $client2->LocalFile ($replica4_ior);
 289 my $client2_replica2_iogr = $client2->LocalFile ($replica2_iogr);
 290
 291 $client2->DeleteFile ($replica3_ior);
 292 $client2->DeleteFile ($replica4_ior);
 293 $client2->DeleteFile ($replica2_iogr);
 294
 295 # Files for client3
 296 my $client3_replica5_ior = $client3->LocalFile ($replica5_ior);
 297 my $client3_replica6_ior = $client3->LocalFile ($replica6_ior);
 298 my $client3_replica3_iogr = $client3->LocalFile ($replica3_iogr);
 299
 300 $client3->DeleteFile ($replica5_ior);
 301 $client3->DeleteFile ($replica6_ior);
 302 $client3->DeleteFile ($replica3_iogr);
 303
 304 $hostname = $rp_manager->HostName ();
 305 $port = $rp_manager->RandomPort ();
 306
 307 $RM = $rp_manager->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager" .
 308                                   "$build_directory/tao_ft_replicationmanager",
 309                                   "-o $rp_manager_ior " .
 310                                   "-ORBEndpoint iiop://$hostname:$port");
 311 my($RMC);
 312 if ($use_rm){
 313     $RMC = $rp_manager_controller->CreateProcess (".$build_directory/replmgr_controller" ,
 314                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 315                                  "-x");
 316 }
 317 else{
 318     $RMC = $rp_manager_controller->CreateProcess (".$build_directory/replmgr_controller" ,
 319                                  "-f file://$rp_manager_controller_registry_ior " .
 320                                  "-x");
 321 }
 322
 323 $REG = $ft_registry->CreateProcess (".$build_directory/ft_registry" ,
 324                                  "-o $ft_registry_registry_ior " .
 325                                  "-q");
 326
 327
 328 my($REP1);
 329 my($REP2);
 330 my($REP3);
 331
 332 if ($use_rm){
 333     $REP1 = $ft_replica1->CreateProcess (".$build_directory/ft_replica" ,
 334                                  "-o $ft_replica1_factory1_ior ".
 335                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 336                                  "-l $location1 -i $role1 -q -p $ft_replica1_client_data");
 337
 338     $REP2 = $ft_replica2->CreateProcess (".$build_directory/ft_replica" ,
 339                                  "-o $ft_replica2_factory2_ior ".
 340                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 341                                  "-l $location2 ".
 342                                  "-i $role1 -i $role2 -i $role3 -q -u " .
 343                                  "-p $ft_replica2_client_data");
 344
 345     $REP3 = $ft_replica3->CreateProcess (".$build_directory/ft_replica" ,
 346                                  "-o $ft_replica3_factory3_ior ".
 347                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 348                                  "-l $location3 -i $role2 -q -u -p $ft_replica1_client_data");
 349
 350     $CTR = $ft_creator->CreateProcess (".$build_directory/ft_create" ,
 351                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 352                                  "-n -r $role1 -r $role2 -r $role1 -u $role3 -i");
 353
 354 }
 355 else{
 356     $REP1 = $ft_replica1->CreateProcess (".$build_directory/ft_replica" ,
 357                                  "-o $ft_replica1_factory1_ior ".
 358                                  "-f file://$ft_replica1_registry_ior " .
 359                                  "-l $location1 -i $role1 -q -p $ft_replica1_client_data");
 360
 361     $REP2 = $ft_replica2->CreateProcess (".$build_directory/ft_replica" ,
 362                                  "-o $ft_replica2_factory2_ior ".
 363                                  "-f file://$ft_replica2_registry_ior " .
 364                                  "-l $location2 ".
 365                                  "-i $role1 -i $role2 -i $role3 -q -u " .
 366                                  "-p $ft_replica2_client_data");
 367
 368     $REP3 = $ft_replica3->CreateProcess (".$build_directory/ft_replica" ,
 369                                  "-o $ft_replica3_factory3_ior ".
 370                                  "-f file://$ft_replica3_registry_ior " .
 371                                  "-l $location3 -i $role2 -q -u -p $ft_replica1_client_data");
 372
 373     $CTR = $ft_creator->CreateProcess (".$build_directory/ft_create" ,
 374                                  "-f file://$ft_creator_registry_ior " .
 375                                  "-n -r $role1 -r $role2 -r $role1 -u $role3 -i");
 376 }
 377
 378 my($CL1) = $client1->CreateProcess (".$build_directory/ft_client" ,
 379                                  "-f file://$client1_replica1_ior " .
 380                                  "-f file://$client1_replica2_ior " .
 381                                  "-c testscript");
 382
 383 my($CL2) = $client2->CreateProcess (".$build_directory/ft_client" ,
 384                                  "-f file://$client2_replica3_ior " .
 385                                  "-f file://$client2_replica4_ior " .
 386                                  "-c testscript");
 387
 388 my($CL3) = $client3->CreateProcess (".$build_directory/ft_client" ,
 389                                  "-f file://$client3_replica5_ior " .
 390                                  "-f file://$client3_replica6_ior " .
 391                                  "-c testscript");
 392
 393
 394 if ($simulated) {
 395     print "\nTEST: Preparing Client Mediated Fault Tolerance test.\n" if ($verbose);
 396 }else{
 397     print "\nTEST: Preparing IOGR based test.\n" if ($verbose);
 398     $CL1->Arguments ("-f file://$client1_replica1_iogr -c testscript");
 399     $CL2->Arguments ("-f file://$client2_replica2_iogr -c testscript");
 400     $CL3->Arguments ("-f file://$client3_replica3_iogr -c testscript");
 401 }
 402
 403 #######################
 404 # Start FactoryRegistry
 405 if ($use_rm){
 406     print "\nTEST: starting ReplicationManager " . $RM->CommandLine . "\n" if ($verbose);
 407     $RM->Spawn ();
 408     add_to_kills ($RM);
 409
 410     print "TEST: waiting for registry's IOR\n" if ($verbose);
 411
 412     if ($rp_manager->WaitForFileTimed ($rm_ior,
 413                                $rp_manager->ProcessStartWaitInterval()) == -1) {
 414         print STDERR "ERROR: cannot find file <$rp_manager_ior>\n";
 415         exit_and_kill (1);
 416     }
 417 }
 418 else{
 419     print "\nTEST: starting registry " . $REG->CommandLine . "\n" if ($verbose);
 420     $REG->Spawn ();
 421
 422     add_to_kills ($REG);
 423
 424     print "TEST: waiting for registry's IOR\n" if ($verbose);
 425
 426     if ($ft_registry->WaitForFileTimed ($registry_ior,
 427                                $ft_registry->ProcessStartWaitInterval()) == -1) {
 428         print STDERR "ERROR: cannot find file <$rp_manager_ior>\n";
 429         exit_and_kill (1);
 430     }
 431
 432     if ($ft_registry->GetFile ($registry_ior) == -1) {
 433         print STDERR "ERROR: cannot retrieve file <$ft_registry_registry_ior\n";
 434         exit_and_kill (1);
 435     }
 436     if ($ft_replica1->PutFile ($registry_ior) == -1) {
 437         print STDERR "ERROR: cannot set file <$ft_replica1_registry_ior>\n";
 438         exit_and_kill (1);
 439     }
 440     if ($ft_replica2->PutFile ($registry_ior) == -1) {
 441         print STDERR "ERROR: cannot set file <$ft_replica2_registry_ior>\n";
 442         exit_and_kill (1);
 443     }
 444     if ($ft_replica3->PutFile ($registry_ior) == -1) {
 445         print STDERR "ERROR: cannot set file <$ft_replica3_registry_ior>\n";
 446         exit_and_kill (1);
 447     }
 448     if ($ft_creator->PutFile ($registry_ior) == -1) {
 449         print STDERR "ERROR: cannot set file <$ft_creator_registry_ior>\n";
 450         exit_and_kill (1);
 451     }
 452 }
 453
 454 #################
 455 # Start Factories
 456
 457 print "\nTEST: starting factory 1 " . $REP1->CommandLine . "\n" if ($verbose);
 458 $REP1->Spawn ();
 459
 460 add_to_kills ($REP1);
 461
 462 print "TEST: waiting for factory 1's IOR\n" if ($verbose);
 463 if ($ft_replica1->WaitForFileTimed ($factory1_ior,
 464                                $ft_replica1->ProcessStartWaitInterval()) == -1) {
 465     print STDERR "ERROR: cannot find file <$ft_replica1_factory1_ior>\n";
 466     exit_and_kill (1);
 467 }
 468
 469 print "\nTEST: starting factory 2 " . $REP2->CommandLine . "\n" if ($verbose);
 470 $REP2->Spawn ();
 471
 472 add_to_kills ($REP2);
 473
 474 print "TEST: waiting for factory 2's IOR\n" if ($verbose);
 475 if ($ft_replica2->WaitForFileTimed ($factory2_ior,
 476                                $ft_replica2->ProcessStartWaitInterval()) == -1) {
 477     print STDERR "ERROR: cannot find file <$ft_replica2_factory2_ior>\n";
 478     exit_and_kill (1);
 479 }
 480
 481 print "\nTEST: starting factory 3 " . $REP3->CommandLine . "\n" if ($verbose);
 482 $REP3->Spawn ();
 483
 484 add_to_kills ($REP3);
 485
 486 print "TEST: waiting for factory 3's IOR\n" if ($verbose);
 487 if ($ft_replica3->WaitForFileTimed ($factory3_ior,
 488                                $ft_replica3->ProcessStartWaitInterval()) == -1) {
 489     print STDERR "ERROR: cannot find file <$ft_replica3_factory3_ior>\n";
 490     exit_and_kill (1);
 491 }
 492 ######################
 493 # Create object groups
 494
 495 print "\nTEST: starting object group creator " . $CTR->CommandLine . "\n" if ($verbose);
 496 $CTR->Spawn ();
 497
 498 add_to_kills ($CTR);
 499
 500 print "TEST: waiting for Replica IOR files from object group creator\n" if ($verbose);
 501
 502 if ($ft_creator->WaitForFileTimed ($replica1_ior,
 503                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 504     print STDERR "ERROR: cannot find file <$ft_creator_replica1_ior>\n";
 505     exit_and_kill (1);
 506 }
 507
 508 if ($ft_creator->WaitForFileTimed ($replica2_ior,
 509                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 510     print STDERR "ERROR: cannot find file <$ft_creator_replica2_ior>\n";
 511     exit_and_kill (1);
 512 }
 513
 514 if ($ft_creator->WaitForFileTimed ($replica3_ior,
 515                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 516     print STDERR "ERROR: cannot find file <$ft_creator_replica3_ior>\n";
 517     exit_and_kill (1);
 518 }
 519
 520 if ($ft_creator->WaitForFileTimed ($replica4_ior,
 521                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 522     print STDERR "ERROR: cannot find file <$ft_creator_replica4_ior>\n";
 523     exit_and_kill (1);
 524 }
 525
 526 if ($ft_creator->WaitForFileTimed ($replica5_ior,
 527                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 528     print STDERR "ERROR: cannot find file <$ft_creator_replica5_ior>\n";
 529     exit_and_kill (1);
 530 }
 531
 532 if ($ft_creator->WaitForFileTimed ($replica6_ior,
 533                                $ft_creator->ProcessStartWaitInterval()) == -1) {
 534     print STDERR "ERROR: cannot find file <$ft_creator_replica6_ior>\n";
 535     exit_and_kill (1);
 536 }
 537
 538 # Get/Put files:
 539 # replica1_ior: client1
 540 if ($ft_creator->GetFile ($replica1_ior) == -1) {
 541     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica1_ior\n";
 542     exit_and_kill (1);
 543 }
 544 if ($client1->PutFile ($replica1_ior) == -1) {
 545     print STDERR "ERROR: cannot set file <$client1_replica1_ior>\n";
 546     exit_and_kill (1);
 547 }
 548 # replica2_ior: client1
 549 if ($ft_creator->GetFile ($replica2_ior) == -1) {
 550     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica2_ior\n";
 551     exit_and_kill (1);
 552 }
 553 if ($client1->PutFile ($replica2_ior) == -1) {
 554     print STDERR "ERROR: cannot set file <$client1_replica2_ior>\n";
 555     exit_and_kill (1);
 556 }
 557 # replica3_ior: client2
 558 if ($ft_creator->GetFile ($replica3_ior) == -1) {
 559     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica3_ior\n";
 560     exit_and_kill (1);
 561 }
 562 if ($client2->PutFile ($replica3_ior) == -1) {
 563     print STDERR "ERROR: cannot set file <$client2_replica3_ior>\n";
 564     exit_and_kill (1);
 565 }
 566 # replica4_ior: client2
 567 if ($ft_creator->GetFile ($replica4_ior) == -1) {
 568     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica4_ior\n";
 569     exit_and_kill (1);
 570 }
 571 if ($client2->PutFile ($replica4_ior) == -1) {
 572     print STDERR "ERROR: cannot set file <$client2_replica4_ior>\n";
 573     exit_and_kill (1);
 574 }
 575 # replica5_ior: client3
 576 if ($ft_creator->GetFile ($replica5_ior) == -1) {
 577     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica5_ior\n";
 578     exit_and_kill (1);
 579 }
 580 if ($client3->PutFile ($replica5_ior) == -1) {
 581     print STDERR "ERROR: cannot set file <$client3_replica5_ior>\n";
 582     exit_and_kill (1);
 583 }
 584 # replica6_ior: client3
 585 if ($ft_creator->GetFile ($replica6_ior) == -1) {
 586     print STDERR "ERROR: cannot retrieve file <$ft_creator_replica6_ior\n";
 587     exit_and_kill (1);
 588 }
 589 if ($client3->PutFile ($replica6_ior) == -1) {
 590     print STDERR "ERROR: cannot set file <$client3_replica6_ior>\n";
 591     exit_and_kill (1);
 592 }
 593
 594 print "\nTEST: wait for object group creator.\n" if ($verbose);
 595 $status_creator = $CTR->WaitKill ($ft_creator->ProcessStopWaitInterval());
 596
 597 if ($status_creator != 0) {
 598     print STDERR "TEST ERROR: configuration manager returned $status_creator\n";
 599     exit_and_kill(1);
 600 }
 601 #############
 602 # Run clients
 603
 604 print "\nTEST: starting client " . $CL1->CommandLine . "\n" if ($verbose);
 605 $status_client = $CL1->SpawnWaitKill ($client1->ProcessStartWaitInterval() + 45);
 606 if ($status_client != 0) {
 607     print STDERR "TEST ERROR: client returned $status_client\n";
 608     exit_and_kill(1);
 609 }
 610
 611 print "\nTEST: starting client again " . $CL2->CommandLine . "\n" if ($verbose);
 612
 613 $status_client = $CL2->SpawnWaitKill ($client2->ProcessStartWaitInterval() + 45);
 614 if ($status_client != 0) {
 615     print STDERR "TEST ERROR: client returned $status_client\n";
 616     exit_and_kill(1);
 617 }
 618
 619 print "\nTEST: starting client, one more time with feeling " . $CL3->CommandLine . "\n" if ($verbose);
 620 $status_client = $CL3->SpawnWaitKill ($client3->ProcessStartWaitInterval() + 45);
 621 if ($status_client != 0) {
 622     print STDERR "TEST ERROR: client returned $status_client\n";
 623     exit_and_kill(1);
 624 }
 625
 626 ######################
 627 # Clean house and exit
 628
 629 print "\nTEST: wait for factory 1.\n" if ($verbose);
 630
 631 $status_replica = $REP1->WaitKill ($ft_replica1->ProcessStopWaitInterval() + 15);
 632 if ($status_replica != 0) {
 633     print STDERR "TEST ERROR: replica returned $status_replica\n";
 634     exit_and_kill(1);
 635 }
 636 print "\nTEST: wait for factory 2.\n" if ($verbose);
 637 $status_replica = $REP2->WaitKill ($ft_replica2->ProcessStopWaitInterval() + 15);
 638 if ($status_replica != 0) {
 639     print STDERR "TEST ERROR: replica returned $status_replica\n";
 640     exit_and_kill(1);
 641 }
 642
 643 print "\nTEST: wait for factory 3.\n" if ($verbose);
 644 $status_replica = $REP3->WaitKill ($ft_replica3->ProcessStopWaitInterval() + 15);
 645 if ($status_replica != 0) {
 646     print STDERR "TEST ERROR: replica returned $status_replica\n";
 647     exit_and_kill(1);
 648 }
 649
 650 if ($use_rm){
 651     print "\nTEST: shutting down the replication manager.\n" if ($verbose);
 652     $status_controller = $RMC->SpawnWaitKill ($rp_manager_controller->ProcessStartWaitInterval() + 285);
 653     if ($status_controller != 0) {
 654         print STDERR "TEST ERROR: replication manager controller returned $status_controller\n";
 655         $status = 1;
 656     }
 657
 658     print "\nTEST: wait for ReplicationManager.\n" if ($verbose);
 659     $status_repmgr = $RM->WaitKill ($rp_manager->ProcessStopWaitInterval() + 15);
 660     if ($status_repmgr != 0) {
 661         print STDERR "TEST ERROR: ReplicationManager returned $status_repmgr\n";
 662         $status = 1;
 663     }
 664 }
 665 else{
 666     print "\nTEST: wait for FactoryRegistry.\n" if ($verbose);
 667     $status_reg = $REG->WaitKill ($ft_registry->ProcessStopWaitInterval() + 15);
 668     if ($status_reg != 0) {
 669         print STDERR "TEST ERROR: FactoryRegistry returned $status_reg\n";
 670         $status = 1;
 671     }
 672 }
 673
 674 print "\nTEST: releasing scratch files.\n" if ($verbose);
 675
 676 # Files for rp_manager
 677 $rp_manager->DeleteFile ($rm_ior);
 678
 679 # Files for ft_registry
 680 $ft_registry->DeleteFile ($registry_ior);
 681
 682 if(!$use_rm){
 683     $rp_manager_controller->DeleteFile ($registry_ior);
 684 }
 685
 686 # Files for ft_replica1
 687 $ft_replica1->DeleteFile ($factory1_ior);
 688 $ft_replica1->DeleteFile ($client_data);
 689
 690 if(!$use_rm){
 691     $ft_replica1->DeleteFile ($registry_ior);
 692 }
 693
 694 # Files for ft_replica2
 695 $ft_replica2->DeleteFile ($factory2_ior);
 696 $ft_replica2->DeleteFile ($client_data);
 697
 698 if(!$use_rm){
 699     $ft_replica2->DeleteFile ($registry_ior);
 700 }
 701 # Files for ft_replica3
 702 $ft_replica3->DeleteFile ($factory3_ior);
 703 $ft_replica3->DeleteFile ($client_data);
 704
 705 if(!$use_rm){
 706     $ft_replica3->DeleteFile ($registry_ior);
 707 }
 708
 709 # Files for ft_creator
 710 if(!$use_rm){
 711     $ft_creator->DeleteFile ($registry_ior);
 712 }
 713
 714 $ft_creator->DeleteFile ($replica1_ior);
 715 $ft_creator->DeleteFile ($replica2_ior);
 716 $ft_creator->DeleteFile ($replica3_ior);
 717 $ft_creator->DeleteFile ($replica4_ior);
 718 $ft_creator->DeleteFile ($replica5_ior);
 719 $ft_creator->DeleteFile ($replica6_ior);
 720
 721 $ft_creator->DeleteFile ($replica1_iogr);
 722 $ft_creator->DeleteFile ($replica2_iogr);
 723 $ft_creator->DeleteFile ($replica3_iogr);
 724
 725 # Files for client1
 726 $client1->DeleteFile ($replica1_ior);
 727 $client1->DeleteFile ($replica2_ior);
 728 $client1->DeleteFile ($replica1_iogr);
 729
 730 # Files for client2
 731 $client2->DeleteFile ($replica3_ior);
 732 $client2->DeleteFile ($replica4_ior);
 733 $client2->DeleteFile ($replica2_iogr);
 734
 735 # Files for client3
 736 $client3->DeleteFile ($replica5_ior);
 737 $client3->DeleteFile ($replica6_ior);
 738 $client3->DeleteFile ($replica3_iogr);
 739
 740 exit $status;