TAO/orbsvcs/tests/FT_Naming/FaultTolerant/run_persistence_test.pl

   1 eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
   2      & eval 'exec perl -S $0 $argv:q'
   3      if 0;
   4
   5 # -*- perl -*-
   6
   7 use lib "$ENV{ACE_ROOT}/bin";
   8 use PerlACE::TestTarget;
   9 use Cwd;
  10 use File::Compare;
  11 use File::Copy;
  12
  13 #$ENV{ACE_TEST_VERBOSE} = "1";
  14
  15 my $startdir = getcwd();
  16 my $debug_level = '0';
  17 my $redirection_enabled = 0;
  18
  19 foreach $i (@ARGV) {
  20     if ($i eq '-debug') {
  21         $debug_level = '10';
  22     }
  23     if ($i eq '-verbose') {
  24         $redirection_enabled = 0;
  25     }
  26 }
  27
  28 my $server  = PerlACE::TestTarget::create_target (1) || die "Create target 1 failed\n";
  29 my $client  = PerlACE::TestTarget::create_target (2) || die "Create target 2 failed\n";
  30 my $server2 = PerlACE::TestTarget::create_target (3) || die "Create target 3 failed\n";
  31
  32 # Variables for command-line arguments to client and server
  33 # executables.
  34
  35 my $stdout_file        = "test.out";
  36 my $stderr_file        = "test.err";
  37 my $client_stdout_file = $client->LocalFile ($stdout_file);
  38 my $client_stderr_file = $client->LocalFile ($stderr_file);
  39 my $client_member_file = "member.dat";
  40
  41 my $status = 0;
  42 my $POSITIVE_TEST_RESULT = 0;
  43 my $NEGATIVE_TEST_RESULT = 1;
  44
  45 my $NSGROUP = $client->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_nsgroup");
  46 my $NSLIST  = $client->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_nslist");
  47 my $NSADD   = $client->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_nsadd");
  48 my $NSDEL   = $client->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_nsdel");
  49
  50 ## Allow the user to determine where the persistence file will be located
  51 ## just in case the current directory is not suitable for locking.
  52 ## We can't change the name of the persistence file because that is not
  53 ## sufficient to work around locking problems for Tru64 when the current
  54 ## directory is NFS mounted from a system that does not properly support
  55 ## locking.
  56 foreach my $possible ($ENV{TMPDIR}, $ENV{TEMP}, $ENV{TMP}) {
  57     if (defined $possible && -d $possible) {
  58       if (chdir($possible)) {
  59         last;
  60       }
  61     }
  62 }
  63
  64 sub cat_file($)
  65 {
  66     my $file_name = shift;
  67     if (-s $file_name ) # size of file is greater than zero
  68     {
  69         open TESTFILE, $file_name or die "Couldn't open file: $!";
  70         my @teststring = <TESTFILE>; # read in all of the file
  71         print STDERR "\n@teststring\n";
  72         close TESTFILE;
  73     }
  74 }
  75
  76 sub redirect_output()
  77 {
  78     open (OLDOUT, ">&", \*STDOUT) or die "Can't dup STDOUT: $!";
  79     open (OLDERR, ">&", \*STDERR) or die "Can't dup STDERR: $!";
  80     open STDERR, '>', $client_stderr_file;
  81     open STDOUT, '>', $client_stdout_file;
  82 }
  83
  84 sub restore_output()
  85 {
  86     open (STDERR, ">&OLDERR") or die "Can't dup OLDERR: $!";
  87     open (STDOUT, ">&OLDOUT") or die "Can't dup OLDOUT: $!";
  88 }
  89
  90 sub compare_file_with_backup($)
  91 {
  92     my $file = shift;
  93     my $backup = $file . ".bak";
  94     unless (-e $backup) {
  95       print STDERR "ERROR: Backup file $backup does not exist\n";
  96       return 1;
  97     }
  98     my $result = compare ($file, $backup);
  99     if ($result != 0) {
 100       print STDERR "ERROR: Backup file $backup does not agree with $file\n";
 101     }
 102     return $result;
 103 }
 104
 105 sub run_nsgroup ($$)
 106 {
 107     my $args = shift;
 108     my $expected_test_result = shift;
 109
 110     my $arglist = "$args";
 111
 112     $NSGROUP->Arguments ($arglist);
 113
 114     if ($redirection_enabled) {
 115         redirect_output();
 116     }
 117
 118     my $nsgroup_status = $NSGROUP->SpawnWaitKill ($client->ProcessStartWaitInterval());
 119
 120     if ($redirection_enabled) {
 121         restore_output();
 122     }
 123
 124     if ($nsgroup_status != $expected_test_result) {
 125         my $time = localtime;
 126         print STDERR "ERROR: nsgroup returned $nsgroup_status at $time\n";
 127         if ($redirection_enabled) {
 128             cat_file($client_stderr_file);
 129             cat_file($client_stdout_file);
 130         }
 131         $status = 1;
 132     }
 133 }
 134
 135 sub run_nslist($$)
 136 {
 137     my $args = shift;
 138     my $expected_test_result = shift;
 139
 140     $NSLIST->Arguments ($args);
 141
 142     if ($redirection_enabled) {
 143         redirect_output();
 144     }
 145
 146     #tao_nslist --ns file://ns.ior
 147     my $nslist_status = $NSLIST->SpawnWaitKill ($client->ProcessStartWaitInterval());
 148
 149     if ($redirection_enabled) {
 150         restore_output();
 151     }
 152
 153     if ($nslist_status != $expected_test_result) {
 154         my $time = localtime;
 155         print STDERR "ERROR: nslist returned $nslist_status at $time\n";
 156         if ($redirection_enabled) {
 157             cat_file($client_stderr_file);
 158             cat_file($client_stdout_file);
 159         }
 160         $status = 1;
 161     }
 162 }
 163
 164 sub run_nsadd($$)
 165 {
 166     my $args = shift;
 167     my $expected_test_result = shift;
 168
 169     $NSADD->Arguments ($args);
 170
 171     if ($redirection_enabled) {
 172         redirect_output();
 173     }
 174
 175     #tao_nsadd --ns file://ns.ior --name iso --ctx
 176     my $nsadd_status = $NSADD->SpawnWaitKill ($client->ProcessStartWaitInterval());
 177
 178     if ($redirection_enabled) {
 179         restore_output();
 180     }
 181
 182     if ($nsadd_status != $expected_test_result) {
 183         my $time = localtime;
 184         print STDERR "ERROR: nsadd returned $nsadd_status at $time\n";
 185         if ($redirection_enabled) {
 186             cat_file($client_stderr_file);
 187             cat_file($client_stdout_file);
 188         }
 189         $status = 1;
 190     }
 191 }
 192
 193 sub run_nsdel($$)
 194 {
 195     my $args = shift;
 196     my $expected_test_result = shift;
 197
 198     $NSDEL->Arguments ($args);
 199
 200     if ($redirection_enabled) {
 201         redirect_output();
 202     }
 203
 204     #tao_nsdel --ns file://ns.ior --name iso --destroy
 205     my $nsdel_status = $NSDEL->SpawnWaitKill ($client->ProcessStartWaitInterval());
 206
 207     if ($redirection_enabled) {
 208         restore_output();
 209     }
 210
 211     if ($nsdel_status != $expected_test_result) {
 212         my $time = localtime;
 213         print STDERR "ERROR: nsdel returned $nsdel_status at $time\n";
 214         if ($redirection_enabled) {
 215             cat_file($client_stderr_file);
 216             cat_file($client_stdout_file);
 217         }
 218         $status = 1;
 219     }
 220 }
 221
 222 sub clean_persistence_dir($$)
 223 {
 224     my $target = shift;
 225     my $directory_name = shift;
 226
 227     chdir $directory_name;
 228     opendir(THISDIR, ".");
 229     @allfiles = grep(!/^\.\.?$/, readdir(THISDIR));
 230     closedir(THISDIR);
 231     foreach $tmp (@allfiles){
 232         $target->DeleteFile ($tmp);
 233     }
 234     chdir "..";
 235 }
 236
 237 # Make sure that the directory to use to hold the naming contexts exists
 238 # and is cleaned out
 239 sub init_persistence_dir($$)
 240 {
 241     my $target = shift;
 242     my $directory_name = shift;
 243
 244     if ( ! -d $directory_name ) {
 245         mkdir ($directory_name, 0777);
 246     } else {
 247         clean_persistence_dir ($target, $directory_name);
 248     }
 249 }
 250
 251 my $name_dir    = "NameService";
 252 my $group_dir   = "GroupService";
 253 my $nm_iorfile  = "nm.ior";
 254 my $ns_iorfile  = "ns.ior";
 255 my $sv_iorfile  = "obj.ior";
 256 my $sv_statusfile = "server.status";
 257
 258 my $ns_ref      = "--ns file://$ns_iorfile";
 259 my $sv2_iorfile = $server2->LocalFile ($sv_iorfile);
 260 my $sv2_statusfile = $server2->LocalFile ($sv_statusfile);
 261
 262
 263
 264
 265 ################################################################################
 266 # setup END block to cleanup after exit call
 267 ################################################################################
 268 END
 269 {
 270     $server->DeleteFile ($ns_iorfile);
 271     $server->DeleteFile ($nm_iorfile);
 272     $client->DeleteFile ($nm_iorfile);
 273     $client->DeleteFile ($stdout_file);
 274     $client->DeleteFile ($stderr_file);
 275     $client->DeleteFile ($client_member_file);
 276     $server2->DeleteFile($sv_iorfile);
 277     $server2->DeleteFile($sv_statusfile);
 278
 279     if ( -d $name_dir ) {
 280         print STDERR "INFO: removing <$name_dir>\n";
 281         clean_persistence_dir ($server, $name_dir);
 282         rmdir ($name_dir);
 283     }
 284
 285     if ( -d $group_dir ) {
 286         print STDERR "INFO: removing <$group_dir>\n";
 287         clean_persistence_dir ($server, $group_dir);
 288         rmdir ($group_dir);
 289     }
 290 }
 291
 292 ################################################################################
 293 # Validate that repository data written by the name service is available upon
 294 # startup.
 295 ################################################################################
 296 sub persistence_test ()
 297 {
 298     my $previous_status = $status;
 299     $status = 0;
 300
 301     my $hostname          = $server->HostName ();
 302     my $ns_orb_port1      = 10001 + $server->RandomPort ();
 303     my $ns_endpoint1      = "iiop://$hostname:$ns_orb_port1";
 304     my $default_init_ref  = "-ORBDefaultInitRef corbaloc:iiop:$hostname:$ns_orb_port1";
 305     my $client_nm_iorfile = $client->LocalFile ($nm_iorfile);
 306
 307     print_msg("Persistence Test");
 308     init_persistence_dir ($server, $name_dir);
 309     init_persistence_dir ($server, $group_dir);
 310
 311     my $ns_args       = "-ORBListenEndPoints $ns_endpoint1 ".
 312                         "-ORBDebugLevel $debug_level " .
 313                         "-g $nm_iorfile ".
 314                         "-o $ns_iorfile ".
 315                         "-v $group_dir ".
 316                         "-u $name_dir ";
 317
 318     my $tao_ft_naming = "$ENV{TAO_ROOT}/orbsvcs/FT_Naming_Service/tao_ft_naming";
 319
 320     my $client1_args = "--persistence " .
 321                        "--create " .
 322                        "-ORBDebugLevel $debug_level " .
 323                        "-p corbaloc:iiop:$hostname:$ns_orb_port1/NameService " .
 324                        "-r corbaloc:iiop:$hostname:$ns_orb_port1/NamingManager " .
 325                        "-b 4 " .
 326                        "-d 4 ";
 327
 328     my $client2_args = "--persistence " .
 329                        "--validate " .
 330                        "-ORBDebugLevel $debug_level " .
 331                        "-p corbaloc:iiop:$hostname:$ns_orb_port1/NameService " .
 332                        "-r corbaloc:iiop:$hostname:$ns_orb_port1/NamingManager " .
 333                        "-b 4 " .
 334                        "-d 4 ";
 335
 336     my $client_prog = "$startdir/client";
 337
 338
 339     ##1. Run one instance of tao_ft_naming service
 340     $NS1 = $server->CreateProcess ($tao_ft_naming, $ns_args);
 341     $CL1 = $client->CreateProcess ($client_prog, $client1_args);
 342     $CL2 = $client->CreateProcess ($client_prog, $client2_args);
 343
 344
 345     my $server2_args = "-ORBdebuglevel $debug_level " .
 346                        "$default_init_ref ".
 347                        "-o $sv2_iorfile ";
 348
 349     $SV2 = $server2->CreateProcess ("$startdir/server", $server2_args);
 350
 351     $server->DeleteFile ($ns_iorfile);
 352     $NS1->Spawn ();
 353     if ($server->WaitForFileTimed ($ns_iorfile,
 354                                    $server->ProcessStartWaitInterval()) == -1) {
 355         print STDERR "ERROR: cannot find file <$ns_iorfile>\n";
 356         $NS1->Kill (); $NS1->TimedWait (1);
 357         exit 1;
 358     }
 359
 360     ##2. Create new contexts and new object groups
 361     print_msg("INFO: starting test server");
 362     $server_status = $SV2->Spawn ();
 363     if ($server_status != 0) {
 364         print STDERR "ERROR: server returned $server_status\n";
 365         exit 1;
 366     }
 367     if ($server2->WaitForFileTimed ($sv_statusfile,
 368                                    $server2->ProcessStartWaitInterval()) == -1) {
 369         print STDERR "ERROR: cannot find file <$sv_statusfile>\n";
 370         $SV2->Kill (); $SV2->TimedWait (1);
 371         exit 1;
 372     }
 373
 374     ##3. Creation additional contexts and object groups and verify
 375     print_msg("INFO: Starting client1");
 376     $client_status = $CL1->SpawnWaitKill ($client->ProcessStartWaitInterval());
 377     if ($client_status != 0) {
 378         print STDERR "ERROR: client1 returned $client_status\n";
 379         $status = 1;
 380     }
 381
 382     ##4. Kill the tao_ft_naming server
 383     print_msg("Kill the tao_ft_naming server");
 384     $server_status = $NS1->TerminateWaitKill ($server->ProcessStopWaitInterval());
 385     if ($server_status != 0) {
 386         print STDERR "ERROR: server 1 returned $server_status\n";
 387         $status = 1;
 388     }
 389
 390     ##5. Start a new instance of the tao_ft_naming server
 391     print_msg("Start a new instance of the tao_ft_naming server");
 392     $server->DeleteFile ($ns_iorfile);
 393     $NS1->Spawn ();
 394     if ($server->WaitForFileTimed ($ns_iorfile,
 395                                    $server->ProcessStartWaitInterval()) == -1) {
 396         print STDERR "ERROR: cannot find file <$ns_iorfile>\n";
 397         $NS1->Kill (); $NS1->TimedWait (1);
 398         $status = 1;
 399     }
 400
 401     ##6. Verify the new name, object group and member are in the tao_ft_naming repository.
 402     print_msg("Verify the new name, object group and member are in the tao_ft_naming repository");
 403     print_msg("INFO: Starting client2");
 404     $client_status = $CL2->SpawnWaitKill ($client->ProcessStartWaitInterval());
 405     if ($client_status != 0) {
 406         print STDERR "ERROR: client2 returned $client_status\n";
 407         $status = 1;
 408     }
 409
 410     print_msg("INFO: terminating test server");
 411     $server_status = $SV2->TerminateWaitKill ($server2->ProcessStopWaitInterval());
 412     if ($server_status != 0) {
 413         print STDERR "ERROR: server returned $server_status\n";
 414         $status = 1;
 415     }
 416
 417     $server_status = $NS1->TerminateWaitKill ($server->ProcessStopWaitInterval());
 418     if ($server_status != 0) {
 419         print STDERR "ERROR: server 1 returned $server_status\n";
 420         $status = 1;
 421     }
 422
 423     if ( $status == 0 ) {
 424         $status = $previous_status;
 425     }
 426
 427     return $status;
 428 }
 429
 430 ################################################################################
 431 # Validate that when a corrupt persistent file is read that the contents of the
 432 # backup file is used instead.
 433 ################################################################################
 434 sub backup_restore_test ()
 435 {
 436
 437     my $num_child_contexts = 8;
 438     my $num_object_groups = 1;
 439
 440     my $previous_status = $status;
 441     $status = 0;
 442
 443     my $hostname          = $server->HostName ();
 444     my $ns_orb_port1      = 10001 + $server->RandomPort ();
 445     my $ns_endpoint1      = "iiop://$hostname:$ns_orb_port1";
 446     my $default_init_ref  = "-ORBDefaultInitRef corbaloc:iiop:$hostname:$ns_orb_port1";
 447     my $client_nm_iorfile = $client->LocalFile ($nm_iorfile);
 448
 449     print_msg("Backup/Restore Test");
 450     init_persistence_dir ($server, $name_dir);
 451     init_persistence_dir ($server, $group_dir);
 452
 453     my $ns_args       = "-ORBListenEndPoints $ns_endpoint1 ".
 454                         "-ORBDebugLevel $debug_level " .
 455                         "-g $nm_iorfile ".
 456                         "-o $ns_iorfile ".
 457                         "-v $group_dir ".
 458                         "-u $name_dir ";
 459
 460     my $tao_ft_naming = "$ENV{TAO_ROOT}/orbsvcs/FT_Naming_Service/tao_ft_naming";
 461
 462     my $client1_args = "--persistence " .
 463                        "--create " .
 464                         "-ORBDebugLevel $debug_level " .
 465                        "-p corbaloc:iiop:$hostname:$ns_orb_port1/NameService " .
 466                        "-r corbaloc:iiop:$hostname:$ns_orb_port1/NamingManager " .
 467                        "-b 4 " .
 468                        "-d 4 ";
 469
 470     my $client2_args = "--persistence " .
 471                        "--validate " .
 472                        "-ORBDebugLevel $debug_level " .
 473                        "-p corbaloc:iiop:$hostname:$ns_orb_port1/NameService " .
 474                        "-r corbaloc:iiop:$hostname:$ns_orb_port1/NamingManager " .
 475                        "-b 4 " .
 476                        "-d 4 ";
 477
 478     my $client_prog = "$startdir/client";
 479
 480
 481     ##1. Run one instance of tao_ft_naming service
 482     $NS1 = $server->CreateProcess ($tao_ft_naming, $ns_args);
 483     $CL1 = $client->CreateProcess ($client_prog, $client1_args);
 484     $CL2 = $client->CreateProcess ($client_prog, $client2_args);
 485
 486
 487     my $server2_args = "-ORBdebuglevel $debug_level " .
 488                        "$default_init_ref ".
 489                        "-o $sv2_iorfile ";
 490
 491     $SV2 = $server2->CreateProcess ("$startdir/server", $server2_args);
 492
 493     $server->DeleteFile ($ns_iorfile);
 494     $NS1->Spawn ();
 495     if ($server->WaitForFileTimed ($ns_iorfile,
 496                                    $server->ProcessStartWaitInterval()) == -1) {
 497         print STDERR "ERROR: cannot find file <$ns_iorfile>\n";
 498         $NS1->Kill (); $NS1->TimedWait (1);
 499         exit 1;
 500     }
 501
 502     ##2. Create new contexts and new object groups
 503     print_msg("INFO: starting test server");
 504     $server_status = $SV2->Spawn ();
 505     if ($server_status != 0) {
 506         print STDERR "ERROR: server returned $server_status\n";
 507         exit 1;
 508     }
 509     if ($server2->WaitForFileTimed ($sv_statusfile,
 510                                    $server2->ProcessStartWaitInterval()) == -1) {
 511         print STDERR "ERROR: cannot find file <$sv_statusfile>\n";
 512         $SV2->Kill (); $SV2->TimedWait (1);
 513         exit 1;
 514     }
 515
 516     ##3. Creation additional contexts and object groups and verify
 517     print_msg("INFO: Starting client1");
 518     $client_status = $CL1->SpawnWaitKill ($client->ProcessStartWaitInterval());
 519     if ($client_status != 0) {
 520         print STDERR "ERROR: client1 returned $client_status\n";
 521         $status = 1;
 522     }
 523
 524     ##4. Kill the tao_ft_naming server
 525     print_msg("Kill the tao_ft_naming server");
 526     $server_status = $NS1->TerminateWaitKill ($server->ProcessStopWaitInterval());
 527     if ($server_status != 0) {
 528         print STDERR "ERROR: server 1 returned $server_status\n";
 529         $status = 1;
 530     }
 531
 532     ##5. Verify that backup files are created
 533
 534     print_msg("Verifying naming context backup files");
 535     $file = $name_dir . "/NameService";
 536     if (compare_file_with_backup ($file) != 0) {
 537       $status = 1;
 538     }
 539     for ($i = 0; $i < $num_child_contexts; $i++) {
 540       $file = $name_dir . "/NameService_$i";
 541       if (compare_file_with_backup ($file) != 0) {
 542           $status = 1;
 543       }
 544     }
 545
 546     print_msg("Verifying object group backup files");
 547     $file = $group_dir . "/ObjectGroup_global";
 548     if (compare_file_with_backup ($file) != 0) {
 549       $status = 1;
 550     }
 551     for ($i = 0; $i < $num_object_groups; $i++) {
 552       $file = $group_dir . "/ObjectGroup_$i";
 553       if (compare_file_with_backup ($file) != 0) {
 554         $status = 1;
 555       }
 556     }
 557
 558     ##6. Replace some of the data files with corrupt files
 559     print_msg("Replace data files with corrupt files");
 560     my $corrupt_data_dir = $startdir . "/corrupt_data/";
 561
 562     my $corrupt_name_dir = $corrupt_data_dir . $name_dir . "/";
 563     opendir(NAMEDIR, $corrupt_name_dir);
 564     @allfiles = grep(/^NameService/, readdir(NAMEDIR));
 565     closedir(NAMEDIR);
 566     foreach $file (@allfiles) {
 567       copy ($corrupt_name_dir . $file, $name_dir . "/" . $file) or die "Copy failed: $!\n";
 568     }
 569
 570     my $corrupt_group_dir = $corrupt_data_dir . $group_dir . "/";
 571     opendir(GROUPDIR, $corrupt_group_dir);
 572     @allfiles = grep(/^ObjectGroup/, readdir(GROUPDIR));
 573     closedir(GROUPDIR);
 574     foreach $file (@allfiles) {
 575       copy ($corrupt_group_dir . $file, $group_dir . "/" . $file) or die "Copy failed: $!\n";
 576     }
 577
 578     ##7. Start a new instance of the tao_ft_naming server
 579     print_msg("Start a new instance of the tao_ft_naming server and then running client 2");
 580     $server->DeleteFile ($ns_iorfile);
 581     # Redirect output so that expected error messages are not interpreted as
 582     # test failure and rely instead of return status.
 583     redirect_output();
 584     my $restore_status = 0;
 585     $NS1->Spawn ();
 586     if ($server->WaitForFileTimed ($ns_iorfile,
 587                                    $server->ProcessStartWaitInterval()) == -1) {
 588         print STDERR "ERROR: cannot find file <$ns_iorfile>\n";
 589         $NS1->Kill (); $NS1->TimedWait (1);
 590         $status = 1;
 591         $restorestatus = 1;
 592     }
 593
 594     ##8. Verify the new name, object group and member are in the tao_ft_naming repository.
 595     print_msg("Verify the backup files are used when the corrupt files are read");
 596     print_msg("INFO: Starting client2");
 597     $client_status = $CL2->SpawnWaitKill ($client->ProcessStartWaitInterval());
 598     restore_output();
 599     if ($client_status != 0) {
 600         print STDERR "ERROR: client2 returned $client_status\n";
 601         $status = 1;
 602         $restorestatus = 1;
 603     }
 604
 605     if ($restore_status == 1) {
 606       cat_file($client_stderr_file);
 607       cat_file($client_stdout_file);
 608     }
 609
 610     print_msg("INFO: terminating test server");
 611     $server_status = $SV2->TerminateWaitKill ($server2->ProcessStopWaitInterval());
 612     if ($server_status != 0) {
 613         print STDERR "ERROR: server returned $server_status\n";
 614         $status = 1;
 615     }
 616
 617     $server_status = $NS1->TerminateWaitKill ($server->ProcessStopWaitInterval());
 618     if ($server_status != 0) {
 619         print STDERR "ERROR: server 1 returned $server_status\n";
 620         $status = 1;
 621     }
 622
 623     if ( $status == 0 ) {
 624         $status = $previous_status;
 625     }
 626
 627     return $status;
 628 }
 629
 630 sub print_msg($)
 631 {
 632     my $msg = shift;
 633     my $bar = "===============================================================================";
 634     print STDERR "\n\n$bar\n$msg\n$bar\n";
 635 }
 636
 637 sub show_result($$)
 638 {
 639     my $test_result = shift;
 640     my $test_name = shift;
 641
 642     if ( 0 == $test_result ) {
 643         print_msg("$test_name: SUCCESS");
 644     } else {
 645         print_msg("$test_name: ERROR");
 646     }
 647 }
 648
 649 my $result = persistence_test ();
 650 my $result = backup_restore_test ();
 651
 652 show_result($result, "Persistence Test");
 653
 654 exit $result;