src/tools/cppunit/TestShell.cpp

   1 #include <iostream>
   2 #include <map>
   3 #include <set>
   4 #include <string>
   5 #include <vector>
   6
   7 #include <Autolock.h>
   8 #include <Directory.h>
   9 #include <Entry.h>
  10 #include <image.h>
  11 #include <Locker.h>
  12 #include <Path.h>
  13 #include <TLS.h>
  14
  15 #include <cppunit/Exception.h>
  16 #include <cppunit/Test.h>
  17 #include <cppunit/TestAssert.h>
  18 #include <cppunit/TestFailure.h>
  19 #include <cppunit/TestResult.h>
  20 #include <cppunit/TestSuite.h>
  21
  22 #include <TestShell.h>
  23 #include <TestListener.h>
  24
  25 using std::cout;
  26 using std::endl;
  27 using std::set;
  28
  29 _EXPORT BTestShell *BTestShell::fGlobalShell = NULL;
  30 const char BTestShell::indent[] = "  ";
  31
  32 _EXPORT
  33 BTestShell::BTestShell(const string &description, SyncObject *syncObject)
  34         : fVerbosityLevel(v2)
  35         , fTestResults(syncObject)
  36         , fDescription(description)
  37         , fListTestsAndExit(false)
  38         , fTestDir(NULL)
  39 #ifndef NO_ELF_SYMBOL_PATCHING
  40         , fPatchGroupLocker(new BLocker)
  41         , fPatchGroup(NULL)
  42         , fOldDebuggerHook(NULL)
  43         , fOldLoadAddOnHook(NULL)
  44         , fOldUnloadAddOnHook(NULL)
  45 #endif // ! NO_ELF_SYMBOL_PATCHING
  46 {
  47         fTLSDebuggerCall = tls_allocate();
  48 }
  49
  50 _EXPORT
  51 BTestShell::~BTestShell() {
  52         delete fTestDir;
  53 #ifndef NO_ELF_SYMBOL_PATCHING
  54         delete fPatchGroupLocker;
  55 #endif // ! NO_ELF_SYMBOL_PATCHING
  56 }
  57
  58
  59 _EXPORT
  60 status_t
  61 BTestShell::AddSuite(BTestSuite *suite) {
  62         if (suite) {
  63                 if (Verbosity() >= v3)
  64                         cout << "Adding suite '" << suite->getName() << "'" << endl;
  65
  66                 // Add the suite
  67                 fSuites[suite->getName()] = suite;
  68
  69                 // Add its tests
  70                 const TestMap &map = suite->getTests();
  71                 for (TestMap::const_iterator i = map.begin();
  72                            i != map.end();
  73                               i++) {
  74                         AddTest(i->first, i->second);
  75                         if (Verbosity() >= v4 && i->second)
  76                                 cout << "  " << i->first << endl;
  77                 }
  78
  79                 return B_OK;
  80         } else
  81                 return B_BAD_VALUE;
  82 }
  83
  84 _EXPORT
  85 void
  86 BTestShell::AddTest(const string &name, CppUnit::Test *test) {
  87         if (test != NULL)
  88                 fTests[name] = test;
  89         else
  90                 fTests.erase(name);
  91 }
  92
  93 _EXPORT
  94 int32
  95 BTestShell::LoadSuitesFrom(BDirectory *libDir) {
  96         if (!libDir || libDir->InitCheck() != B_OK)
  97                 return 0;
  98
  99         BEntry addonEntry;
 100         BPath addonPath;
 101         image_id addonImage;
 102         int count = 0;
 103
 104         typedef BTestSuite* (*suiteFunc)(void);
 105         suiteFunc func;
 106
 107         while (libDir->GetNextEntry(&addonEntry, true) == B_OK) {
 108                 status_t err;
 109                 err = addonEntry.GetPath(&addonPath);
 110                 if (!err) {
 111 //                      cout << "Checking " << addonPath.Path() << "..." << endl;
 112                         addonImage = load_add_on(addonPath.Path());
 113                         err = (addonImage >= 0 ? B_OK : B_ERROR);
 114                 }
 115                 if (err == B_OK) {
 116 //                      cout << "..." << endl;
 117                         err = get_image_symbol(addonImage, "getTestSuite",
 118                                 B_SYMBOL_TYPE_TEXT, reinterpret_cast<void **>(&func));
 119                 } else {
 120 //                      cout << " !!! err == " << err << endl;
 121                 }
 122                 if (err == B_OK)
 123                         err = AddSuite(func());
 124                 if (err == B_OK)
 125                         count++;
 126         }
 127         return count;
 128 }
 129
 130 _EXPORT
 131 int
 132 BTestShell::Run(int argc, char *argv[]) {
 133         // Make note of which directory we started in
 134         UpdateTestDir(argv);
 135
 136         // Parse the command line args
 137         if (!ProcessArguments(argc, argv))
 138                 return 0;
 139
 140         // Load any dynamically loadable tests we can find
 141         LoadDynamicSuites();
 142
 143         // See if the user requested a list of tests. If so,
 144         // print and bail.
 145         if (fListTestsAndExit) {
 146                 PrintInstalledTests();
 147                 return 0;
 148         }
 149
 150         // Add the proper tests to our suite (or exit if there
 151         // are no tests installed).
 152         CppUnit::TestSuite suite;
 153         if (fTests.empty()) {
 154
 155                 // No installed tests whatsoever, so bail
 156                 cout << "ERROR: No installed tests to run!" << endl;
 157                 return 0;
 158
 159         } else if (fSuitesToRun.empty() && fTestsToRun.empty()) {
 160
 161                 // None specified, so run them all
 162                 TestMap::iterator i;
 163                 for (i = fTests.begin(); i != fTests.end(); ++i)
 164                         suite.addTest( i->second );
 165
 166         } else {
 167                 set<string>::const_iterator i;
 168                 set<string> suitesToRemove;
 169
 170                 // Add all the tests from any specified suites to the list of
 171                 // tests to run (since we use a set, this eliminates the concern
 172                 // of having duplicate entries).
 173                 for (i = fTestsToRun.begin(); i != fTestsToRun.end(); ++i) {
 174                         // See if it's a suite (since it may just be a single test)
 175                         if (fSuites.find(*i) != fSuites.end()) {
 176                                 // Note the suite name for later removal unless the
 177                                 // name is also the name of an available individual test
 178                                 if (fTests.find(*i) == fTests.end()) {
 179                                         suitesToRemove.insert(*i);
 180                                 }
 181                                 const TestMap &tests = fSuites[*i]->getTests();
 182                                 TestMap::const_iterator j;
 183                                 for (j = tests.begin(); j != tests.end(); j++) {
 184                                         fTestsToRun.insert( j->first );
 185                                 }
 186                         }
 187                 }
 188
 189                 // Remove the names of all of the suites we discovered from the
 190                 // list of tests to run (unless there's also an installed individual
 191                 // test of the same name).
 192                 for (i = suitesToRemove.begin(); i != suitesToRemove.end(); i++) {
 193                         fTestsToRun.erase(*i);
 194                 }
 195
 196                 // Everything still in fTestsToRun must then be an explicit test
 197                 for (i = fTestsToRun.begin(); i != fTestsToRun.end(); ++i) {
 198                         // Make sure it's a valid test
 199                         if (fTests.find(*i) != fTests.end()) {
 200                                 suite.addTest( fTests[*i] );
 201                         } else {
 202                                 cout << endl << "ERROR: Invalid argument \"" << *i << "\"" << endl;
 203                                 PrintHelp();
 204                                 return 0;
 205                         }
 206                 }
 207
 208         }
 209
 210         // Run all the tests
 211         InitOutput();
 212         InstallPatches();
 213         suite.run(&fTestResults);
 214         UninstallPatches();
 215         PrintResults();
 216
 217         return 0;
 218 }
 219
 220 _EXPORT
 221 BTestShell::VerbosityLevel
 222 BTestShell::Verbosity() const {
 223         return fVerbosityLevel;
 224 }
 225
 226 _EXPORT
 227 const char*
 228 BTestShell::TestDir() const {
 229         return (fTestDir ? fTestDir->Path() : NULL);
 230 }
 231
 232 // ExpectDebuggerCall
 233 /*!     \brief Marks the current thread as ready for a debugger() call.
 234
 235         A subsequent call of debugger() will be intercepted and a respective
 236         flag will be set. WasDebuggerCalled() will then return \c true.
 237 */
 238 _EXPORT
 239 void
 240 BTestShell::ExpectDebuggerCall()
 241 {
 242         void *var = tls_get(fTLSDebuggerCall);
 243         ::CppUnit::Asserter::failIf(var, "ExpectDebuggerCall(): Already expecting "
 244                 "a debugger() call.");
 245         tls_set(fTLSDebuggerCall, (void*)1);
 246 }
 247
 248 // WasDebuggerCalled
 249 /*!     \brief Returns whether the current thread has invoked debugger() since
 250                    the last ExpectDebuggerCall() invocation and resets the mode so
 251                    that subsequent debugger() calls will hit the debugger.
 252         \return \c true, if debugger() has been called by the current thread since
 253                         the last invocation of ExpectDebuggerCall(), \c false otherwise.
 254 */
 255 _EXPORT
 256 bool
 257 BTestShell::WasDebuggerCalled()
 258 {
 259         void *var = tls_get(fTLSDebuggerCall);
 260         tls_set(fTLSDebuggerCall, NULL);
 261         return ((addr_t)var > 1);
 262 }
 263
 264 _EXPORT
 265 void
 266 BTestShell::PrintDescription(int argc, char *argv[]) {
 267         cout << endl << fDescription;
 268 }
 269
 270 _EXPORT
 271 void
 272 BTestShell::PrintHelp() {
 273         cout << endl;
 274         cout << "VALID ARGUMENTS:     " << endl;
 275         PrintValidArguments();
 276         cout << endl;
 277
 278 }
 279
 280 _EXPORT
 281 void
 282 BTestShell::PrintValidArguments() {
 283         cout << indent << "--help       Displays this help text plus some other garbage" << endl;
 284         cout << indent << "--list       Lists the names of classes with installed tests" << endl;
 285         cout << indent << "-v0          Sets verbosity level to 0 (concise summary only)" << endl;
 286         cout << indent << "-v1          Sets verbosity level to 1 (complete summary only)" << endl;
 287         cout << indent << "-v2          Sets verbosity level to 2 (*default* -- per-test results plus" << endl;
 288         cout << indent << "             complete summary)" << endl;
 289         cout << indent << "-v3          Sets verbosity level to 3 (partial dynamic loading information, " << endl;
 290         cout << indent << "             per-test results and timing info, plus complete summary)" << endl;
 291         cout << indent << "-v4          Sets verbosity level to 4 (complete dynamic loading information, " << endl;
 292         cout << indent << "             per-test results and timing info, plus complete summary)" << endl;
 293         cout << indent << "NAME         Instructs the program to run the test for the given class or all" << endl;
 294         cout << indent << "             the tests for the given suite. If some bonehead adds both a class" << endl;
 295         cout << indent << "             and a suite with the same name, the suite will be run, not the class" << endl;
 296         cout << indent << "             (unless the class is part of the suite with the same name :-). If no" << endl;
 297         cout << indent << "             classes or suites are specified, all available tests are run" << endl;
 298         cout << indent << "-lPATH       Adds PATH to the search path for dynamically loadable test" << endl;
 299         cout << indent << "             libraries" << endl;
 300 }
 301
 302 _EXPORT
 303 void
 304 BTestShell::PrintInstalledTests() {
 305         // Print out the list of installed suites
 306         cout << "------------------------------------------------------------------------------" << endl;
 307         cout << "Available Suites:" << endl;
 308         cout << "------------------------------------------------------------------------------" << endl;
 309         SuiteMap::const_iterator j;
 310         for (j = fSuites.begin(); j != fSuites.end(); ++j)
 311                 cout << j->first << endl;
 312         cout << endl;
 313
 314         // Print out the list of installed tests
 315         cout << "------------------------------------------------------------------------------" << endl;
 316         cout << "Available Tests:" << endl;
 317         cout << "------------------------------------------------------------------------------" << endl;
 318         TestMap::const_iterator i;
 319         for (i = fTests.begin(); i != fTests.end(); ++i)
 320                 cout << i->first << endl;
 321         cout << endl;
 322 }
 323
 324 _EXPORT
 325 bool
 326 BTestShell::ProcessArguments(int argc, char *argv[]) {
 327         // If we're given no parameters, the default settings
 328         // will do just fine
 329         if (argc < 2)
 330                 return true;
 331
 332         // Handle each command line argument (skipping the first
 333         // which is just the app name)
 334         for (int i = 1; i < argc; i++) {
 335                 string str(argv[i]);
 336
 337                 if (!ProcessArgument(str, argc, argv))
 338                         return false;
 339         }
 340
 341         return true;
 342 }
 343
 344 _EXPORT
 345 bool
 346 BTestShell::ProcessArgument(string arg, int argc, char *argv[]) {
 347         if (arg == "--help") {
 348                 PrintDescription(argc, argv);
 349                 PrintHelp();
 350                 return false;
 351         } else if (arg == "--list") {
 352                 fListTestsAndExit = true;
 353         } else if (arg == "-v0") {
 354                 fVerbosityLevel = v0;
 355         } else if (arg == "-v1") {
 356                 fVerbosityLevel = v1;
 357         } else if (arg == "-v2") {
 358                 fVerbosityLevel = v2;
 359         } else if (arg == "-v3") {
 360                 fVerbosityLevel = v3;
 361         } else if (arg == "-v4") {
 362                 fVerbosityLevel = v4;
 363         } else if (arg.length() >= 2 && arg[0] == '-' && arg[1] == 'l') {
 364                 fLibDirs.insert(arg.substr(2, arg.size()-2));
 365         } else {
 366                 fTestsToRun.insert(arg);
 367         }
 368         return true;
 369 }
 370
 371 _EXPORT
 372 void
 373 BTestShell::InitOutput() {
 374         // For vebosity level 2, we output info about each test
 375         // as we go. This involves a custom CppUnit::TestListener
 376         // class.
 377         if (fVerbosityLevel >= v2) {
 378                 cout << "------------------------------------------------------------------------------" << endl;
 379                 cout << "Tests" << endl;
 380                 cout << "------------------------------------------------------------------------------" << endl;
 381                 fTestResults.addListener(new BTestListener);
 382         }
 383         fTestResults.addListener(&fResultsCollector);
 384 }
 385
 386 _EXPORT
 387 void
 388 BTestShell::PrintResults() {
 389
 390         if (fVerbosityLevel > v0) {
 391                 // Print out detailed results for verbosity levels > 0
 392                 cout << "------------------------------------------------------------------------------" << endl;
 393                 cout << "Results " << endl;
 394                 cout << "------------------------------------------------------------------------------" << endl;
 395
 396                 // Print failures and errors if there are any, otherwise just say "PASSED"
 397                 ::CppUnit::TestResultCollector::TestFailures::const_iterator iFailure;
 398                 if (fResultsCollector.testFailuresTotal() > 0) {
 399                         if (fResultsCollector.testFailures() > 0) {
 400                                 cout << "- FAILURES: " << fResultsCollector.testFailures() << endl;
 401                                 for (iFailure = fResultsCollector.failures().begin();
 402                                      iFailure != fResultsCollector.failures().end();
 403                                      ++iFailure)
 404                                 {
 405                                         if (!(*iFailure)->isError())
 406                                                 cout << "    " << (*iFailure)->toString() << endl;
 407                                 }
 408                         }
 409                         if (fResultsCollector.testErrors() > 0) {
 410                                 cout << "- ERRORS: " << fResultsCollector.testErrors() << endl;
 411                                 for (iFailure = fResultsCollector.failures().begin();
 412                                      iFailure != fResultsCollector.failures().end();
 413                                      ++iFailure)
 414                                 {
 415                                         if ((*iFailure)->isError())
 416                                                 cout << "    " << (*iFailure)->toString() << endl;
 417                                 }
 418                         }
 419
 420                 }
 421                 else
 422                         cout << "+ PASSED" << endl;
 423
 424                 cout << endl;
 425
 426         }
 427         else {
 428                 // Print out concise results for verbosity level == 0
 429                 if (fResultsCollector.testFailuresTotal() > 0)
 430                         cout << "- FAILED" << endl;
 431                 else
 432                         cout << "+ PASSED" << endl;
 433         }
 434
 435 }
 436
 437 _EXPORT
 438 void
 439 BTestShell::LoadDynamicSuites() {
 440         if (Verbosity() >= v3) {
 441                 cout << "------------------------------------------------------------------------------" << endl;
 442                 cout << "Loading " << endl;
 443                 cout << "------------------------------------------------------------------------------" << endl;
 444         }
 445
 446         set<string>::iterator i;
 447         for (i = fLibDirs.begin(); i != fLibDirs.end(); i++) {
 448                 BDirectory libDir((*i).c_str());
 449                 if (Verbosity() >= v3)
 450                         cout << "Checking " << *i << endl;
 451 /*              int count =*/ LoadSuitesFrom(&libDir);
 452                 if (Verbosity() >= v3) {
 453 //                      cout << "Loaded " << count << " suite" << (count == 1 ? "" : "s");
 454 //                      cout << " from " << *i << endl;
 455                 }
 456         }
 457
 458         if (Verbosity() >= v3)
 459                 cout << endl;
 460
 461         // Look for suites and tests with the same name and give a
 462         // warning, as this is only asking for trouble... :-)
 463         for (SuiteMap::const_iterator i = fSuites.begin(); i != fSuites.end(); i++) {
 464                 if (fTests.find(i->first) != fTests.end() && Verbosity() > v0) {
 465                         cout << "WARNING: '" << i->first << "' refers to both a test suite *and* an individual" <<
 466                         endl << "         test. Both will be executed, but it is reccommended you rename" <<
 467                         endl << "         one of them to resolve the conflict." <<
 468                         endl << endl;
 469                 }
 470         }
 471
 472 }
 473
 474 _EXPORT
 475 void
 476 BTestShell::UpdateTestDir(char *argv[]) {
 477         BPath path(argv[0]);
 478         if (path.InitCheck() == B_OK) {
 479                 delete fTestDir;
 480                 fTestDir = new BPath();
 481                 if (path.GetParent(fTestDir) != B_OK)
 482                         cout << "Couldn't get test dir." << endl;
 483         } else
 484                 cout << "Couldn't find the path to the test app." << endl;
 485 }
 486
 487 // InstallPatches
 488 /*!     \brief Patches the debugger() function.
 489
 490         load_add_on() and unload_add_on() are patches as well, to keep the
 491         patch group up to date, when images are loaded/unloaded.
 492 */
 493 _EXPORT
 494 void
 495 BTestShell::InstallPatches()
 496 {
 497 #ifndef NO_ELF_SYMBOL_PATCHING
 498         if (fPatchGroup) {
 499                 std::cerr << "BTestShell::InstallPatches(): Patch group already exist!"
 500                         << endl;
 501                 return;
 502         }
 503         BAutolock locker(fPatchGroupLocker);
 504         if (!locker.IsLocked()) {
 505                 std::cerr << "BTestShell::InstallPatches(): Failed to acquire patch "
 506                         "group lock!" << endl;
 507                 return;
 508         }
 509         fPatchGroup = new(std::nothrow) ElfSymbolPatchGroup;
 510         // init the symbol patch group
 511         if (!fPatchGroup) {
 512                 std::cerr << "BTestShell::InstallPatches(): Failed to allocate patch "
 513                         "group!" << endl;
 514                 return;
 515         }
 516         if (// debugger()
 517                 fPatchGroup->AddPatch("debugger", (void*)&_DebuggerHook,
 518                                                           (void**)&fOldDebuggerHook) == B_OK
 519                 // load_add_on()
 520                 && fPatchGroup->AddPatch("load_add_on", (void*)&_LoadAddOnHook,
 521                                                                  (void**)&fOldLoadAddOnHook) == B_OK
 522                 // unload_add_on()
 523                 && fPatchGroup->AddPatch("unload_add_on", (void*)&_UnloadAddOnHook,
 524                                                                  (void**)&fOldUnloadAddOnHook) == B_OK
 525                 ) {
 526                 // everything went fine
 527                 fPatchGroup->Patch();
 528         } else {
 529                 std::cerr << "BTestShell::InstallPatches(): Failed to patch all "
 530                         "symbols!" << endl;
 531                 UninstallPatches();
 532         }
 533 #endif // ! NO_ELF_SYMBOL_PATCHING
 534 }
 535
 536 // UninstallPatches
 537 /*!     \brief Undoes the patches applied by InstallPatches().
 538 */
 539 _EXPORT
 540 void
 541 BTestShell::UninstallPatches()
 542 {
 543 #ifndef NO_ELF_SYMBOL_PATCHING
 544         BAutolock locker(fPatchGroupLocker);
 545         if (!locker.IsLocked()) {
 546                 std::cerr << "BTestShell::UninstallPatches(): "
 547                         "Failed to acquire patch group lock!" << endl;
 548                 return;
 549         }
 550         if (fPatchGroup) {
 551                 fPatchGroup->Restore();
 552                 delete fPatchGroup;
 553                 fPatchGroup = NULL;
 554         }
 555 #endif // ! NO_ELF_SYMBOL_PATCHING
 556 }
 557
 558 #ifndef NO_ELF_SYMBOL_PATCHING
 559
 560 // _Debugger
 561 _EXPORT
 562 void
 563 BTestShell::_Debugger(const char *message)
 564 {
 565         if (!this || !fPatchGroup) {
 566                 debugger(message);
 567                 return;
 568         }
 569         BAutolock locker(fPatchGroupLocker);
 570         if (!locker.IsLocked() || !fPatchGroup) {
 571                 debugger(message);
 572                 return;
 573         }
 574 cout << "debugger() called: " << message << endl;
 575         void *var = tls_get(fTLSDebuggerCall);
 576         if (var)
 577                 tls_set(fTLSDebuggerCall, (void*)((int)var + 1));
 578         else
 579                 (*fOldDebuggerHook)(message);
 580 }
 581
 582 // _LoadAddOn
 583 _EXPORT
 584 image_id
 585 BTestShell::_LoadAddOn(const char *path)
 586 {
 587         if (!this || !fPatchGroup)
 588                 return load_add_on(path);
 589         BAutolock locker(fPatchGroupLocker);
 590         if (!locker.IsLocked() || !fPatchGroup)
 591                 return load_add_on(path);
 592         image_id result = (*fOldLoadAddOnHook)(path);
 593         fPatchGroup->Update();
 594         return result;
 595 }
 596
 597 // _UnloadAddOn
 598 _EXPORT
 599 status_t
 600 BTestShell::_UnloadAddOn(image_id image)
 601 {
 602         if (!this || !fPatchGroup)
 603                 return unload_add_on(image);
 604         BAutolock locker(fPatchGroupLocker);
 605         if (!locker.IsLocked() || !fPatchGroup)
 606                 return unload_add_on(image);
 607
 608         if (!this || !fPatchGroup)
 609                 return unload_add_on(image);
 610         status_t result = (*fOldUnloadAddOnHook)(image);
 611         fPatchGroup->Update();
 612         return result;
 613 }
 614
 615 // _DebuggerHook
 616 _EXPORT
 617 void
 618 BTestShell::_DebuggerHook(const char *message)
 619 {
 620         fGlobalShell->_Debugger(message);
 621 }
 622
 623 // _LoadAddOnHook
 624 _EXPORT
 625 image_id
 626 BTestShell::_LoadAddOnHook(const char *path)
 627 {
 628         return fGlobalShell->_LoadAddOn(path);
 629 }
 630
 631 // _UnloadAddOnHook
 632 _EXPORT
 633 status_t
 634 BTestShell::_UnloadAddOnHook(image_id image)
 635 {
 636         return fGlobalShell->_UnloadAddOn(image);
 637 }
 638
 639 #endif // ! NO_ELF_SYMBOL_PATCHING