TAO/orbsvcs/tests/FT_App/run_test_rmnotifier.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2      & eval 'exec perl -S $0 $argv:q'
   3     if 0;
   4
   5 # -*- perl -*-
   6
   7 # Purpose:
   8 #       To test the FaultNotifier
   9 #
  10 # Process being tested:
  11 #       Fault_Notifier
  12 #           implements FaultNotifier interface.
  13 # Processes used in test:
  14 #       ReplicationManager
  15 #
  16 #       FT_Replica
  17 #           implements TestReplica interface.
  18 #           implements PullMonitorable.
  19 #       Fault_Detector
  20 #         implements FaultDetectorFactory interface
  21 #         implements PullMonitorable interface
  22 #       FT_Client
  23 #           client for TestReplica interface.
  24 #           client for PullMonitorable.
  25 #       StubAnalyzer
  26 #           Subscribes to Fault_Notfier
  27 #
  28 # Test Scenario (***Test: marks behavior being tested):
  29 #   Phase 1:
  30 #     Start two FT_Replicas
  31 #       FT_Replicas write TestReplica IORs (FR#1 and FR#2) to files
  32 #     Start the Fault_Detector
  33 #       Fault_Detector writes its IOR (FDF) to a file
  34 #     Start the Fault_Notifier
  35 #       Fault_Notifier writes its IOR (FN) to a file.
  36 #   Phase 2:
  37 #     Wait for IORs: FR#1, FR#2, FDF, and FN
  38 #     Start the StubAnalyzer giving it IORS: FR#1, FR#2 FDF, FN
  39 #      StubAnalyzer calls FDF to create a FaultDetector
  40 #       for each Replica.
  41 #      StubAnalyzer subscribes to Fault_Notifier
  42 #      StubAnalyzer writes dummy message(READY) to a file.
  43 #   Phase 3:
  44 #     Wait for READY
  45 #     Start FT_Client giving it IORS: FR#1 and FR#2. [1]
  46 #       FT_Client interacts with FR#1.
  47 #       FT_Client asks FR#1 to fault.  It does so.
  48 #       FT_Client notices fault and switches to FR#2. [1]
  49 #       FD#1 notices fault and notifies Fault_Notifier
  50 #       FD#1 terminates
  51 #       ***Test: Fault_Notifier forwards notification to StubAnalyzer
  52 #       StubAnalyzer prints notification.
  53 #       FT_Client interacts with FR#2.
  54 #       FT_Client asks FR#2 to shut down.
  55 #       FT_Client shuts down.
  56 #       FD#2 notices FR2 is gone, interprets this
  57 #         as a fault, and sends notification to Fault_Notifier
  58 #       FD#2 terminates.
  59 #       ***Test: Fault_Notifier forwards notification to StubAnalyzer
  60 #       StubAnalyzer prints notification.
  61 #     Phase 4: shutting down.
  62 #       All FaultDetectors have terminated so the FaultDetectorFactory
  63 #        honors the "quit-on-idle" option on it's command line and exits.
  64 #       StubAnalyzer compares # fault notifications to # replicas. When
  65 #        they match, it "knows" that the test is over, so it shuts down.
  66 #        As it does so, it disconnects its fault consumers from the FaultNotifier.
  67 #       FaultNotifier notices the last fault consumer disconnecting and exits because
  68 #        the "quit-on-idle" option was specified on the command line.
  69 #     Phase 5: housekeeping
  70 #       Wait for all processes to terminate.
  71 #       Check termination status.
  72 #       Delete temp files.
  73 #
  74 # [1] Client mediated fault tolerance.  These points will
  75 #     change when IOGR support is available.
  76 use lib "$ENV{ACE_ROOT}/bin";
  77 use PerlACE::TestTarget;
  78
  79 ########################
  80 #command line options
  81 #set defaults:
  82
  83 my($status) = 0;
  84 my($verbose) = 0;         # 1: report perl actions before executing them
  85 my($debug_builds) = 0;    # 0: use exes from Release directories
  86 my($simulated) = 1;       # 1: use "client simulated" fault tolerance
  87
  88
  89 # List of the proccess which must be killed before exit with error
  90 @kill_list = ();
  91
  92 sub add_to_kills{
  93     $goal = shift;
  94     @kill_list = (@kill_list, $goal);
  95 }
  96 sub exit_and_kill{
  97     $status = shift;
  98     foreach $goal (@kill_list){
  99         $goal->Kill (); $goal->TimedWait (1);
 100     }
 101     exit $status;
 102 }
 103
 104 foreach $i (@ARGV) {
 105     if ($i eq "--debug_build"){
 106         $debug_builds = 1;
 107     }
 108     elsif ($i eq "--no_simulate"){  # reverse this once we have FT ORB support
 109         $simulated = 0;
 110     }
 111     elsif ($i eq "-v"){
 112         $verbose += 1;
 113     }
 114 }
 115
 116
 117 my($build_directory) = "/Release";
 118 if ( $debug_builds ) {
 119     $build_directory = "";
 120 }
 121
 122 if ( $verbose > 1) {
 123     print "verbose: $verbose\n";
 124     print "debug_builds: $debug_builds -> $build_directory\n";
 125     print "simulated: $simulated\n";
 126 }
 127
 128 my $rp_manager = PerlACE::TestTarget::create_target (1) || die "Create target 1 failed\n";
 129 my $rp_manager_controller = PerlACE::TestTarget::create_target (2) || die "Create target 2 failed\n";
 130 my $ft_replica1 = PerlACE::TestTarget::create_target (3) || die "Create target 3 failed\n";
 131 my $ft_replica2 = PerlACE::TestTarget::create_target (4) || die "Create target 4 failed\n";
 132 my $fault_detector = PerlACE::TestTarget::create_target (5) || die "Create target 5 failed\n";
 133 my $fault_notifier = PerlACE::TestTarget::create_target (6) || die "Create target 6 failed\n";
 134 my $fault_analyzer = PerlACE::TestTarget::create_target (7) || die "Create target 7 failed\n";
 135 my $client = PerlACE::TestTarget::create_target (8) || die "Create target 8 failed\n";
 136
 137 #define temp files
 138
 139 $hostname = $rp_manager->HostName ();
 140 $port = $rp_manager->RandomPort ();
 141
 142 my($rm_ior) = "rm.ior";
 143 my($factory1_ior) = "factory1.ior";
 144 my($factory2_ior) = "factory2.ior";
 145 my($replica1_ior) = "replica1.ior";
 146 my($replica2_ior) = "replica2.ior";
 147 my($detector_ior) = "detector.ior";
 148 my($notifier_ior) = "notifier.ior";
 149 my($client_data) = "persistent.dat";
 150 my($ready_file) = "ready.file";
 151
 152 # Files for rp_manager
 153 my $rp_manager_ior = $rp_manager->LocalFile ($rm_ior);
 154 $rp_manager->DeleteFile ($rm_ior);
 155
 156 # Files for ft_replica1
 157 my $ft_replica1_factory1_ior = $ft_replica1->LocalFile ($factory1_ior);
 158 my $ft_replica1_replica1_ior = $ft_replica1->LocalFile ($replica1_ior);
 159 my $ft_replica1_client_data = $ft_replica1->LocalFile ($client_data);
 160
 161 $ft_replica1->DeleteFile ($factory1_ior);
 162 $ft_replica1->DeleteFile ($replica1_ior);
 163 $ft_replica1->DeleteFile ($client_data);
 164
 165 # Files for ft_replica2
 166 my $ft_replica2_factory2_ior = $ft_replica2->LocalFile ($factory2_ior);
 167 my $ft_replica2_replica2_ior = $ft_replica2->LocalFile ($replica2_ior);
 168 my $ft_replica2_client_data = $ft_replica2->LocalFile ($client_data);
 169
 170 $ft_replica2->DeleteFile ($factory2_ior);
 171 $ft_replica2->DeleteFile ($replica2_ior);
 172 $ft_replica2->DeleteFile ($client_data);
 173
 174 # Files for fault_detector
 175 my $fault_detector_ior = $fault_detector->LocalFile ($detector_ior);
 176 $fault_detector->DeleteFile ($detector_ior);
 177
 178 # Files for fault_notifier
 179 my $fault_notifier_ior = $fault_notifier->LocalFile ($notifier_ior);
 180 $fault_notifier->DeleteFile ($notifier_ior);
 181
 182 # Files for fault_analyzer
 183 my $analyzer_replica1_ior = $fault_analyzer->LocalFile ($replica1_ior);
 184 my $analyzer_replica2_ior = $fault_analyzer->LocalFile ($replica2_ior);
 185 my $analyzer_notifier_ior = $fault_analyzer->LocalFile ($notifier_ior);
 186 my $analyzer_detector_ior = $fault_analyzer->LocalFile ($detector_ior);
 187 my $analyzer_ready_file = $fault_analyzer->LocalFile ($ready_file);
 188
 189 $fault_analyzer->DeleteFile ($replica1_ior);
 190 $fault_analyzer->DeleteFile ($replica2_ior);
 191 $fault_analyzer->DeleteFile ($notifier_ior);
 192 $fault_analyzer->DeleteFile ($detector_ior);
 193 $fault_analyzer->DeleteFile ($ready_file);
 194
 195 # Files for client
 196 my $client_replica1_ior = $client->LocalFile ($replica1_ior);
 197 my $client_replica2_ior = $client->LocalFile ($replica2_ior);
 198
 199 $client->DeleteFile ($replica1_ior);
 200 $client->DeleteFile ($replica2_ior);
 201
 202
 203 ### Create process
 204 $RM = $rp_manager->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager" .
 205                                   "$build_directory/tao_ft_replicationmanager",
 206                                   "-o $rp_manager_ior " .
 207                                   "-ORBEndpoint iiop://$hostname:$port");
 208
 209 $RMC = $rp_manager_controller->CreateProcess (".$build_directory/replmgr_controller" ,
 210                                  "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 211                                  "-x");
 212
 213 $REP1 = $ft_replica1->CreateProcess (".$build_directory/ft_replica" ,
 214                                  "-o $ft_replica1_factory1_ior ".
 215                                  "-f none -t $ft_replica1_replica1_ior " .
 216                                  "-l loc1 -i type1 -q -p $ft_replica1_client_data");
 217
 218 $REP2 = $ft_replica2->CreateProcess (".$build_directory/ft_replica" ,
 219                                      "-o $ft_replica2_factory2_ior ".
 220                                      "-f none -t $ft_replica2_replica2_ior " .
 221                                      "-l loc2 -i type1 -q -p $ft_replica2_client_data");
 222
 223 $DET = $fault_detector->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector" .
 224                                        "$build_directory/tao_fault_detector" ,
 225                                        "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 226                                        "-o $fault_detector_ior -q");
 227
 228 $NOT = $fault_notifier->CreateProcess ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier" .
 229                                        "$build_directory/tao_fault_notifier" ,
 230                                         "-ORBInitRef ReplicationManager=corbaloc::$hostname:$port/ReplicationManager " .
 231                                         "-o $fault_notifier_ior -q");
 232
 233 $ANA = $fault_analyzer->CreateProcess (".$build_directory/ft_analyzer",
 234                                         "-o $analyzer_ready_file " .
 235                                         "-n file://$analyzer_notifier_ior " .
 236                                         "-d file://$analyzer_detector_ior " .
 237                                         "-r file://$analyzer_replica1_ior " .
 238                                         "-r file://$analyzer_replica2_ior -q");
 239
 240 my($CL);
 241 if ($simulated) {
 242     print "\nTEST: Preparing Client Mediated Fault Tolerance test.\n" if ($verbose);
 243     $CL = $client->CreateProcess (".$build_directory/ft_client",
 244                                   "-f file://$client_replica1_ior " .
 245                                   "-f file://$client_replica2_ior " .
 246                                   "-c testscript");
 247
 248 }else{
 249     print "\nTEST: Preparing IOGR based test.\n" if ($verbose);
 250     $CL = $client->CreateProcess (".$build_directory/ft_client",
 251                                   "-f file://$client_replica1_ior " .
 252                                   "-c testscript");
 253 }
 254
 255 #######################
 256 # ReplicationManager
 257
 258 print "\nTEST: starting ReplicationManager " . $RM->CommandLine . "\n" if ($verbose);
 259 $RM->Spawn ();
 260
 261 add_to_kills ($RM);
 262
 263 print "TEST: waiting for registry's IOR\n" if ($verbose);
 264
 265 if ($rp_manager->WaitForFileTimed ($rm_ior,
 266                                $rp_manager->ProcessStartWaitInterval()) == -1) {
 267     print STDERR "ERROR: cannot find file <$rp_manager_ior>\n";
 268     exit_and_kill (1);
 269 }
 270
 271 ##########
 272 # Notifier
 273 print "\nTEST: starting notifier " . $NOT->CommandLine . "\n" if ($verbose);
 274 $NOT->Spawn ();
 275
 276 add_to_kills ($NOT);
 277
 278 print "TEST: waiting for notifier's IOR\n" if ($verbose);
 279 if ($fault_notifier->WaitForFileTimed ($notifier_ior,
 280                                $fault_notifier->ProcessStartWaitInterval()) == -1) {
 281     print STDERR "ERROR: cannot find file <$fault_notifier_ior>\n";
 282     exit_and_kill (1);
 283 }
 284 # Get/Put $notifier_ior: $analyzer_notifier_ior
 285 if ($fault_notifier->GetFile ($notifier_ior) == -1) {
 286     print STDERR "ERROR: cannot retrieve file <$fault_notifier_ior>\n";
 287     exit_and_kill (1);
 288 }
 289 if ($fault_analyzer->PutFile ($notifier_ior) == -1) {
 290     print STDERR "ERROR: cannot set file <$analyzer_notifier_ior>\n";
 291     exit_and_kill (1);
 292 }
 293
 294 ##########
 295 # Detector
 296 print "\nTEST: starting detector factory " . $DET->CommandLine . "\n" if ($verbose);
 297
 298 $DET->Spawn ();
 299
 300 add_to_kills ($DET);
 301
 302 print "TEST: waiting for detector's IOR\n" if ($verbose);
 303 if ($fault_detector->WaitForFileTimed ($detector_ior,
 304                                $fault_detector->ProcessStartWaitInterval()) == -1) {
 305     print STDERR "ERROR: cannot find file <$fault_detector_ior>\n";
 306     exit_and_kill (1);
 307 }
 308
 309 # Get/Put $detector_ior: $analyzer_detector_ior
 310 if ($fault_detector->GetFile ($detector_ior) == -1) {
 311     print STDERR "ERROR: cannot retrieve file <$fault_detector_ior>\n";
 312     exit_and_kill (1);
 313 }
 314 if ($fault_analyzer->PutFile ($detector_ior) == -1) {
 315     print STDERR "ERROR: cannot set file <$analyzer_detector_ior>\n";
 316     exit_and_kill (1);
 317 }
 318
 319
 320 ###########
 321 # Replica 1
 322 print "\nTEST: starting replica1 " . $REP1->CommandLine . "\n" if ($verbose);
 323
 324 $REP1->Spawn ();
 325
 326 add_to_kills ($REP1);
 327
 328 print "TEST: waiting for replica 1's IOR\n" if ($verbose);
 329 if ($ft_replica1->WaitForFileTimed ($replica1_ior,
 330                                $ft_replica1->ProcessStartWaitInterval()) == -1) {
 331     print STDERR "ERROR: cannot find file <$ft_replica1_replica1_ior>\n";
 332     exit_and_kill (1);
 333 }
 334 # Get/Put $replica1_ior: analyzer_replica1_ior $client_replica1_ior
 335 if ($ft_replica1->GetFile ($replica1_ior) == -1) {
 336     print STDERR "ERROR: cannot retrieve file <$ft_replica1_replica1_ior>\n";
 337     exit_and_kill (1);
 338 }
 339 if ($fault_analyzer->PutFile ($replica1_ior) == -1) {
 340     print STDERR "ERROR: cannot set file <$analyzer_replica1_ior>\n";
 341     exit_and_kill (1);
 342 }
 343 if ($client->PutFile ($replica1_ior) == -1) {
 344     print STDERR "ERROR: cannot set file <$client_replica1_ior>\n";
 345     exit_and_kill (1);
 346 }
 347
 348 ###########
 349 # Replica 2
 350 print "\nTEST: starting replica2 " . $REP2->CommandLine . "\n" if ($verbose);
 351 $REP2->Spawn ();
 352
 353 add_to_kills ($REP2);
 354
 355 print "TEST: waiting for replica 2's IOR\n" if ($verbose);
 356 if ($ft_replica2->WaitForFileTimed ($replica2_ior,
 357                                $ft_replica2->ProcessStartWaitInterval()) == -1) {
 358     print STDERR "TEST ERROR: cannot find file <$ft_replica2_replica2_ior>\n";
 359     exit_and_kill (1);
 360 }
 361
 362 # Get/Put $replica2_ior: analyzer_replica2_ior $client_replica2_ior
 363 if ($ft_replica2->GetFile ($replica2_ior) == -1) {
 364     print STDERR "ERROR: cannot retrieve file <$ft_replica2_replica2_ior>\n";
 365     exit_and_kill (1);
 366 }
 367 if ($fault_analyzer->PutFile ($replica2_ior) == -1) {
 368     print STDERR "ERROR: cannot set file <$analyzer_replica2_ior>\n";
 369     exit_and_kill (1);
 370 }
 371 if ($client->PutFile ($replica2_ior) == -1) {
 372     print STDERR "ERROR: cannot set file <$client_replica2_ior>\n";
 373     exit_and_kill (1);
 374 }
 375
 376 ##########
 377 # Analyzer
 378 print "\nTEST: starting analyzer " . $ANA->CommandLine . "\n" if ($verbose);
 379 $ANA->Spawn ();
 380
 381 add_to_kills ($ANA);
 382
 383 print "TEST: waiting for READY.FILE from analyzer\n" if ($verbose);
 384
 385 if ($fault_analyzer->WaitForFileTimed ($ready_file,
 386                                $fault_analyzer->ProcessStartWaitInterval()) == -1) {
 387     print STDERR "TEST ERROR: cannot find file <$analyzer_ready_file>\n";
 388     exit_and_kill (1);
 389 }
 390
 391 ########
 392 # Client
 393 print "\nTEST: starting client " . $CL->CommandLine . "\n" if ($verbose);
 394
 395 $status = $CL->SpawnWaitKill ($client->ProcessStartWaitInterval() + 45);
 396
 397 if ($status != 0) {
 398     print STDERR "TEST ERROR: client returned $status\n";
 399     $status = 1;
 400 }
 401
 402 print "\nTEST: wait for replica 1.\n" if ($verbose);
 403
 404 $status = $REP1->WaitKill ($ft_replica1->ProcessStopWaitInterval());
 405 if ($status != 0) {
 406     print STDERR "TEST ERROR: replica returned $status\n";
 407     $status = 1;
 408 }
 409 print "\nTEST: wait for replica 2.\n" if ($verbose);
 410
 411 $status = $REP2->WaitKill ($ft_replica2->ProcessStopWaitInterval());
 412 if ($status != 0) {
 413     print STDERR "TEST ERROR: replica returned $status\n";
 414     $status = 1;
 415 }
 416
 417 print "\nTEST: wait for detector factory to leave.\n" if ($verbose);
 418 $status = $DET->WaitKill ($fault_detector->ProcessStopWaitInterval());
 419 if ($status != 0) {
 420     print STDERR "TEST ERROR: detector returned $status\n";
 421     $status = 1;
 422 }
 423
 424 print "\nTEST: wait for analyzer to leave.\n" if ($verbose);
 425 $status = $ANA->WaitKill ($fault_analyzer->ProcessStopWaitInterval());
 426 if ($status != 0) {
 427     print STDERR "TEST ERROR: analyzer returned $status\n";
 428     $status = 1;
 429 }
 430
 431 print "\nTEST: shutting down the replication manager.\n" if ($verbose);
 432 $status = $RMC->SpawnWaitKill ($rp_manager_controller->ProcessStartWaitInterval() + 285);
 433 if ($status != 0) {
 434     print STDERR "TEST ERROR: replication manager controller returned $status\n";
 435     $status = 1;
 436 }
 437
 438 print "\nTEST: wait for ReplicationManager.\n" if ($verbose);
 439 $status = $RM->WaitKill ($rp_manager->ProcessStopWaitInterval());
 440 if ($status != 0) {
 441     print STDERR "TEST ERROR: ReplicationManager returned $status\n";
 442     $status = 1;
 443 }
 444
 445 print "\nTEST: wait for notifier to leave.\n" if ($verbose);
 446 $status = $NOT->WaitKill ($fault_notifier->ProcessStopWaitInterval());
 447 if ($status != 0) {
 448     print STDERR "TEST ERROR: notifier returned $status\n";
 449     $status = 1;
 450 }
 451
 452 print "\nTEST: releasing scratch files.\n" if ($verbose);
 453
 454 # Files for rp_manager
 455 $rp_manager->DeleteFile ($rm_ior);
 456
 457 # Files for ft_replica1
 458 $ft_replica1->DeleteFile ($factory1_ior);
 459 $ft_replica1->DeleteFile ($replica1_ior);
 460 $ft_replica1->DeleteFile ($client_data);
 461
 462 # Files for ft_replica2
 463 $ft_replica2->DeleteFile ($factory2_ior);
 464 $ft_replica2->DeleteFile ($replica2_ior);
 465 $ft_replica2->DeleteFile ($client_data);
 466
 467 # Files for fault_detector
 468 $fault_detector->DeleteFile ($detector_ior);
 469
 470 # Files for fault_notifier
 471 $fault_notifier->DeleteFile ($notifier_ior);
 472
 473 # Files for fault_analyzer
 474 $fault_analyzer->DeleteFile ($replica1_ior);
 475 $fault_analyzer->DeleteFile ($replica2_ior);
 476 $fault_analyzer->DeleteFile ($notifier_ior);
 477 $fault_analyzer->DeleteFile ($detector_ior);
 478 $fault_analyzer->DeleteFile ($ready_file);
 479
 480 # Files for client
 481 $client->DeleteFile ($replica1_ior);
 482 $client->DeleteFile ($replica2_ior);
 483
 484 exit $status;