src/tools/cppunit/TestShell.cpp

   1 #include <iostream>
   2 #include <map>
   3 #include <set>
   4 #include <string>
   5 #include <vector>
   6
   7 #include <Autolock.h>
   8 #include <Directory.h>
   9 #include <Entry.h>
  10 #include <image.h>
  11 #include <Locker.h>
  12 #include <Path.h>
  13 #include <TLS.h>
  14
  15 #include <cppunit/Exception.h>
  16 #include <cppunit/Test.h>
  17 #include <cppunit/TestAssert.h>
  18 #include <cppunit/TestFailure.h>
  19 #include <cppunit/TestResult.h>
  20 #include <cppunit/TestSuite.h>
  21
  22 #include <TestShell.h>
  23 #include <TestListener.h>
  24
  25 #ifndef NO_ELF_SYMBOL_PATCHING
  26 #       include <ElfSymbolPatcher.h>
  27 #endif
  28
  29 using std::cout;
  30 using std::endl;
  31 using std::set;
  32
  33 _EXPORT BTestShell *BTestShell::fGlobalShell = NULL;
  34 const char BTestShell::indent[] = "  ";
  35
  36 _EXPORT
  37 BTestShell::BTestShell(const string &description, SyncObject *syncObject)
  38         : fVerbosityLevel(v2)
  39         , fTestResults(syncObject)
  40         , fDescription(description)
  41         , fListTestsAndExit(false)
  42         , fTestDir(NULL)
  43 #ifndef NO_ELF_SYMBOL_PATCHING
  44         , fPatchGroupLocker(new(nothrow) BLocker)
  45         , fPatchGroup(NULL)
  46         , fOldDebuggerHook(NULL)
  47         , fOldLoadAddOnHook(NULL)
  48         , fOldUnloadAddOnHook(NULL)
  49 #endif // ! NO_ELF_SYMBOL_PATCHING
  50 {
  51         fTLSDebuggerCall = tls_allocate();
  52 }
  53
  54 _EXPORT
  55 BTestShell::~BTestShell() {
  56         delete fTestDir;
  57 #ifndef NO_ELF_SYMBOL_PATCHING
  58         delete fPatchGroupLocker;
  59 #endif // ! NO_ELF_SYMBOL_PATCHING
  60 }
  61
  62
  63 _EXPORT
  64 status_t
  65 BTestShell::AddSuite(BTestSuite *suite) {
  66         if (suite) {
  67                 if (Verbosity() >= v3)
  68                         cout << "Adding suite '" << suite->getName() << "'" << endl;
  69
  70                 // Add the suite
  71                 fSuites[suite->getName()] = suite;
  72
  73                 // Add its tests
  74                 const TestMap &map = suite->getTests();
  75                 for (TestMap::const_iterator i = map.begin();
  76                            i != map.end();
  77                               i++) {
  78                         AddTest(i->first, i->second);
  79                         if (Verbosity() >= v4 && i->second)
  80                                 cout << "  " << i->first << endl;
  81                 }
  82
  83                 return B_OK;
  84         } else
  85                 return B_BAD_VALUE;
  86 }
  87
  88 _EXPORT
  89 void
  90 BTestShell::AddTest(const string &name, CppUnit::Test *test) {
  91         if (test != NULL)
  92                 fTests[name] = test;
  93         else
  94                 fTests.erase(name);
  95 }
  96
  97 _EXPORT
  98 int32
  99 BTestShell::LoadSuitesFrom(BDirectory *libDir) {
 100         if (!libDir || libDir->InitCheck() != B_OK)
 101                 return 0;
 102
 103         BEntry addonEntry;
 104         BPath addonPath;
 105         image_id addonImage;
 106         int count = 0;
 107
 108         typedef BTestSuite* (*suiteFunc)(void);
 109         suiteFunc func;
 110
 111         while (libDir->GetNextEntry(&addonEntry, true) == B_OK) {
 112                 status_t err;
 113                 err = addonEntry.GetPath(&addonPath);
 114                 if (!err) {
 115 //                      cout << "Checking " << addonPath.Path() << "..." << endl;
 116                         addonImage = load_add_on(addonPath.Path());
 117                         err = (addonImage >= 0 ? B_OK : B_ERROR);
 118                 }
 119                 if (err == B_OK) {
 120 //                      cout << "..." << endl;
 121                         err = get_image_symbol(addonImage, "getTestSuite",
 122                                 B_SYMBOL_TYPE_TEXT, reinterpret_cast<void **>(&func));
 123                 } else {
 124 //                      cout << " !!! err == " << err << endl;
 125                 }
 126                 if (err == B_OK)
 127                         err = AddSuite(func());
 128                 if (err == B_OK)
 129                         count++;
 130         }
 131         return count;
 132 }
 133
 134 _EXPORT
 135 int
 136 BTestShell::Run(int argc, char *argv[]) {
 137         // Make note of which directory we started in
 138         UpdateTestDir(argv);
 139
 140         // Parse the command line args
 141         if (!ProcessArguments(argc, argv))
 142                 return 0;
 143
 144         // Load any dynamically loadable tests we can find
 145         LoadDynamicSuites();
 146
 147         // See if the user requested a list of tests. If so,
 148         // print and bail.
 149         if (fListTestsAndExit) {
 150                 PrintInstalledTests();
 151                 return 0;
 152         }
 153
 154         // Add the proper tests to our suite (or exit if there
 155         // are no tests installed).
 156         CppUnit::TestSuite suite;
 157         if (fTests.empty()) {
 158
 159                 // No installed tests whatsoever, so bail
 160                 cout << "ERROR: No installed tests to run!" << endl;
 161                 return 0;
 162
 163         } else if (fSuitesToRun.empty() && fTestsToRun.empty()) {
 164
 165                 // None specified, so run them all
 166                 TestMap::iterator i;
 167                 for (i = fTests.begin(); i != fTests.end(); ++i)
 168                         suite.addTest( i->second );
 169
 170         } else {
 171                 set<string>::const_iterator i;
 172                 set<string> suitesToRemove;
 173
 174                 // Add all the tests from any specified suites to the list of
 175                 // tests to run (since we use a set, this eliminates the concern
 176                 // of having duplicate entries).
 177                 for (i = fTestsToRun.begin(); i != fTestsToRun.end(); ++i) {
 178                         // See if it's a suite (since it may just be a single test)
 179                         if (fSuites.find(*i) != fSuites.end()) {
 180                                 // Note the suite name for later removal unless the
 181                                 // name is also the name of an available individual test
 182                                 if (fTests.find(*i) == fTests.end()) {
 183                                         suitesToRemove.insert(*i);
 184                                 }
 185                                 const TestMap &tests = fSuites[*i]->getTests();
 186                                 TestMap::const_iterator j;
 187                                 for (j = tests.begin(); j != tests.end(); j++) {
 188                                         fTestsToRun.insert( j->first );
 189                                 }
 190                         }
 191                 }
 192
 193                 // Remove the names of all of the suites we discovered from the
 194                 // list of tests to run (unless there's also an installed individual
 195                 // test of the same name).
 196                 for (i = suitesToRemove.begin(); i != suitesToRemove.end(); i++) {
 197                         fTestsToRun.erase(*i);
 198                 }
 199
 200                 // Everything still in fTestsToRun must then be an explicit test
 201                 for (i = fTestsToRun.begin(); i != fTestsToRun.end(); ++i) {
 202                         // Make sure it's a valid test
 203                         if (fTests.find(*i) != fTests.end()) {
 204                                 suite.addTest( fTests[*i] );
 205                         } else {
 206                                 cout << endl << "ERROR: Invalid argument \"" << *i << "\"" << endl;
 207                                 PrintHelp();
 208                                 return 0;
 209                         }
 210                 }
 211
 212         }
 213
 214         // Run all the tests
 215         InitOutput();
 216         InstallPatches();
 217         suite.run(&fTestResults);
 218         UninstallPatches();
 219         PrintResults();
 220
 221         return 0;
 222 }
 223
 224 _EXPORT
 225 BTestShell::VerbosityLevel
 226 BTestShell::Verbosity() const {
 227         return fVerbosityLevel;
 228 }
 229
 230 _EXPORT
 231 const char*
 232 BTestShell::TestDir() const {
 233         return (fTestDir ? fTestDir->Path() : NULL);
 234 }
 235
 236 // ExpectDebuggerCall
 237 /*!     \brief Marks the current thread as ready for a debugger() call.
 238
 239         A subsequent call of debugger() will be intercepted and a respective
 240         flag will be set. WasDebuggerCalled() will then return \c true.
 241 */
 242 _EXPORT
 243 void
 244 BTestShell::ExpectDebuggerCall()
 245 {
 246         void *var = tls_get(fTLSDebuggerCall);
 247         ::CppUnit::Asserter::failIf(var, "ExpectDebuggerCall(): Already expecting "
 248                 "a debugger() call.");
 249         tls_set(fTLSDebuggerCall, (void*)1);
 250 }
 251
 252 // WasDebuggerCalled
 253 /*!     \brief Returns whether the current thread has invoked debugger() since
 254                    the last ExpectDebuggerCall() invocation and resets the mode so
 255                    that subsequent debugger() calls will hit the debugger.
 256         \return \c true, if debugger() has been called by the current thread since
 257                         the last invocation of ExpectDebuggerCall(), \c false otherwise.
 258 */
 259 _EXPORT
 260 bool
 261 BTestShell::WasDebuggerCalled()
 262 {
 263         void *var = tls_get(fTLSDebuggerCall);
 264         tls_set(fTLSDebuggerCall, NULL);
 265         return ((addr_t)var > 1);
 266 }
 267
 268 _EXPORT
 269 void
 270 BTestShell::PrintDescription(int argc, char *argv[]) {
 271         cout << endl << fDescription;
 272 }
 273
 274 _EXPORT
 275 void
 276 BTestShell::PrintHelp() {
 277         cout << endl;
 278         cout << "VALID ARGUMENTS:     " << endl;
 279         PrintValidArguments();
 280         cout << endl;
 281
 282 }
 283
 284 _EXPORT
 285 void
 286 BTestShell::PrintValidArguments() {
 287         cout << indent << "--help       Displays this help text plus some other garbage" << endl;
 288         cout << indent << "--list       Lists the names of classes with installed tests" << endl;
 289         cout << indent << "-v0          Sets verbosity level to 0 (concise summary only)" << endl;
 290         cout << indent << "-v1          Sets verbosity level to 1 (complete summary only)" << endl;
 291         cout << indent << "-v2          Sets verbosity level to 2 (*default* -- per-test results plus" << endl;
 292         cout << indent << "             complete summary)" << endl;
 293         cout << indent << "-v3          Sets verbosity level to 3 (partial dynamic loading information, " << endl;
 294         cout << indent << "             per-test results and timing info, plus complete summary)" << endl;
 295         cout << indent << "-v4          Sets verbosity level to 4 (complete dynamic loading information, " << endl;
 296         cout << indent << "             per-test results and timing info, plus complete summary)" << endl;
 297         cout << indent << "NAME         Instructs the program to run the test for the given class or all" << endl;
 298         cout << indent << "             the tests for the given suite. If some bonehead adds both a class" << endl;
 299         cout << indent << "             and a suite with the same name, the suite will be run, not the class" << endl;
 300         cout << indent << "             (unless the class is part of the suite with the same name :-). If no" << endl;
 301         cout << indent << "             classes or suites are specified, all available tests are run" << endl;
 302         cout << indent << "-lPATH       Adds PATH to the search path for dynamically loadable test" << endl;
 303         cout << indent << "             libraries" << endl;
 304 }
 305
 306 _EXPORT
 307 void
 308 BTestShell::PrintInstalledTests() {
 309         // Print out the list of installed suites
 310         cout << "------------------------------------------------------------------------------" << endl;
 311         cout << "Available Suites:" << endl;
 312         cout << "------------------------------------------------------------------------------" << endl;
 313         SuiteMap::const_iterator j;
 314         for (j = fSuites.begin(); j != fSuites.end(); ++j)
 315                 cout << j->first << endl;
 316         cout << endl;
 317
 318         // Print out the list of installed tests
 319         cout << "------------------------------------------------------------------------------" << endl;
 320         cout << "Available Tests:" << endl;
 321         cout << "------------------------------------------------------------------------------" << endl;
 322         TestMap::const_iterator i;
 323         for (i = fTests.begin(); i != fTests.end(); ++i)
 324                 cout << i->first << endl;
 325         cout << endl;
 326 }
 327
 328 _EXPORT
 329 bool
 330 BTestShell::ProcessArguments(int argc, char *argv[]) {
 331         // If we're given no parameters, the default settings
 332         // will do just fine
 333         if (argc < 2)
 334                 return true;
 335
 336         // Handle each command line argument (skipping the first
 337         // which is just the app name)
 338         for (int i = 1; i < argc; i++) {
 339                 string str(argv[i]);
 340
 341                 if (!ProcessArgument(str, argc, argv))
 342                         return false;
 343         }
 344
 345         return true;
 346 }
 347
 348 _EXPORT
 349 bool
 350 BTestShell::ProcessArgument(string arg, int argc, char *argv[]) {
 351         if (arg == "--help") {
 352                 PrintDescription(argc, argv);
 353                 PrintHelp();
 354                 return false;
 355         } else if (arg == "--list") {
 356                 fListTestsAndExit = true;
 357         } else if (arg == "-v0") {
 358                 fVerbosityLevel = v0;
 359         } else if (arg == "-v1") {
 360                 fVerbosityLevel = v1;
 361         } else if (arg == "-v2") {
 362                 fVerbosityLevel = v2;
 363         } else if (arg == "-v3") {
 364                 fVerbosityLevel = v3;
 365         } else if (arg == "-v4") {
 366                 fVerbosityLevel = v4;
 367         } else if (arg.length() >= 2 && arg[0] == '-' && arg[1] == 'l') {
 368                 fLibDirs.insert(arg.substr(2, arg.size()-2));
 369         } else {
 370                 fTestsToRun.insert(arg);
 371         }
 372         return true;
 373 }
 374
 375 _EXPORT
 376 void
 377 BTestShell::InitOutput() {
 378         // For vebosity level 2, we output info about each test
 379         // as we go. This involves a custom CppUnit::TestListener
 380         // class.
 381         if (fVerbosityLevel >= v2) {
 382                 cout << "------------------------------------------------------------------------------" << endl;
 383                 cout << "Tests" << endl;
 384                 cout << "------------------------------------------------------------------------------" << endl;
 385                 fTestResults.addListener(new BTestListener);
 386         }
 387         fTestResults.addListener(&fResultsCollector);
 388 }
 389
 390 _EXPORT
 391 void
 392 BTestShell::PrintResults() {
 393
 394         if (fVerbosityLevel > v0) {
 395                 // Print out detailed results for verbosity levels > 0
 396                 cout << "------------------------------------------------------------------------------" << endl;
 397                 cout << "Results " << endl;
 398                 cout << "------------------------------------------------------------------------------" << endl;
 399
 400                 // Print failures and errors if there are any, otherwise just say "PASSED"
 401                 ::CppUnit::TestResultCollector::TestFailures::const_iterator iFailure;
 402                 if (fResultsCollector.testFailuresTotal() > 0) {
 403                         if (fResultsCollector.testFailures() > 0) {
 404                                 cout << "- FAILURES: " << fResultsCollector.testFailures() << endl;
 405                                 for (iFailure = fResultsCollector.failures().begin();
 406                                      iFailure != fResultsCollector.failures().end();
 407                                      ++iFailure)
 408                                 {
 409                                         if (!(*iFailure)->isError())
 410                                                 cout << "    " << (*iFailure)->toString() << endl;
 411                                 }
 412                         }
 413                         if (fResultsCollector.testErrors() > 0) {
 414                                 cout << "- ERRORS: " << fResultsCollector.testErrors() << endl;
 415                                 for (iFailure = fResultsCollector.failures().begin();
 416                                      iFailure != fResultsCollector.failures().end();
 417                                      ++iFailure)
 418                                 {
 419                                         if ((*iFailure)->isError())
 420                                                 cout << "    " << (*iFailure)->toString() << endl;
 421                                 }
 422                         }
 423
 424                 }
 425                 else
 426                         cout << "+ PASSED" << endl;
 427
 428                 cout << endl;
 429
 430         }
 431         else {
 432                 // Print out concise results for verbosity level == 0
 433                 if (fResultsCollector.testFailuresTotal() > 0)
 434                         cout << "- FAILED" << endl;
 435                 else
 436                         cout << "+ PASSED" << endl;
 437         }
 438
 439 }
 440
 441 _EXPORT
 442 void
 443 BTestShell::LoadDynamicSuites() {
 444         if (Verbosity() >= v3) {
 445                 cout << "------------------------------------------------------------------------------" << endl;
 446                 cout << "Loading " << endl;
 447                 cout << "------------------------------------------------------------------------------" << endl;
 448         }
 449
 450         set<string>::iterator i;
 451         for (i = fLibDirs.begin(); i != fLibDirs.end(); i++) {
 452                 BDirectory libDir((*i).c_str());
 453                 if (Verbosity() >= v3)
 454                         cout << "Checking " << *i << endl;
 455 /*              int count =*/ LoadSuitesFrom(&libDir);
 456                 if (Verbosity() >= v3) {
 457 //                      cout << "Loaded " << count << " suite" << (count == 1 ? "" : "s");
 458 //                      cout << " from " << *i << endl;
 459                 }
 460         }
 461
 462         if (Verbosity() >= v3)
 463                 cout << endl;
 464
 465         // Look for suites and tests with the same name and give a
 466         // warning, as this is only asking for trouble... :-)
 467         for (SuiteMap::const_iterator i = fSuites.begin(); i != fSuites.end(); i++) {
 468                 if (fTests.find(i->first) != fTests.end() && Verbosity() > v0) {
 469                         cout << "WARNING: '" << i->first << "' refers to both a test suite *and* an individual" <<
 470                         endl << "         test. Both will be executed, but it is reccommended you rename" <<
 471                         endl << "         one of them to resolve the conflict." <<
 472                         endl << endl;
 473                 }
 474         }
 475
 476 }
 477
 478 _EXPORT
 479 void
 480 BTestShell::UpdateTestDir(char *argv[]) {
 481         BPath path(argv[0]);
 482         if (path.InitCheck() == B_OK) {
 483                 delete fTestDir;
 484                 fTestDir = new BPath();
 485                 if (path.GetParent(fTestDir) != B_OK)
 486                         cout << "Couldn't get test dir." << endl;
 487         } else
 488                 cout << "Couldn't find the path to the test app." << endl;
 489 }
 490
 491 // InstallPatches
 492 /*!     \brief Patches the debugger() function.
 493
 494         load_add_on() and unload_add_on() are patches as well, to keep the
 495         patch group up to date, when images are loaded/unloaded.
 496 */
 497 _EXPORT
 498 void
 499 BTestShell::InstallPatches()
 500 {
 501 #ifndef NO_ELF_SYMBOL_PATCHING
 502         if (fPatchGroup) {
 503                 cerr << "BTestShell::InstallPatches(): Patch group already exist!"
 504                         << endl;
 505                 return;
 506         }
 507         BAutolock locker(fPatchGroupLocker);
 508         if (!locker.IsLocked()) {
 509                 cerr << "BTestShell::InstallPatches(): Failed to acquire patch "
 510                         "group lock!" << endl;
 511                 return;
 512         }
 513         fPatchGroup = new(nothrow) ElfSymbolPatchGroup;
 514         // init the symbol patch group
 515         if (!fPatchGroup) {
 516                 cerr << "BTestShell::InstallPatches(): Failed to allocate patch "
 517                         "group!" << endl;
 518                 return;
 519         }
 520         if (// debugger()
 521                 fPatchGroup->AddPatch("debugger", (void*)&_DebuggerHook,
 522                                                           (void**)&fOldDebuggerHook) == B_OK
 523                 // load_add_on()
 524                 && fPatchGroup->AddPatch("load_add_on", (void*)&_LoadAddOnHook,
 525                                                                  (void**)&fOldLoadAddOnHook) == B_OK
 526                 // unload_add_on()
 527                 && fPatchGroup->AddPatch("unload_add_on", (void*)&_UnloadAddOnHook,
 528                                                                  (void**)&fOldUnloadAddOnHook) == B_OK
 529                 ) {
 530                 // everything went fine
 531                 fPatchGroup->Patch();
 532         } else {
 533                 cerr << "BTestShell::InstallPatches(): Failed to patch all symbols!"
 534                         << endl;
 535                 UninstallPatches();
 536         }
 537 #endif // ! NO_ELF_SYMBOL_PATCHING
 538 }
 539
 540 // UninstallPatches
 541 /*!     \brief Undoes the patches applied by InstallPatches().
 542 */
 543 _EXPORT
 544 void
 545 BTestShell::UninstallPatches()
 546 {
 547 #ifndef NO_ELF_SYMBOL_PATCHING
 548         BAutolock locker(fPatchGroupLocker);
 549         if (!locker.IsLocked()) {
 550                 cerr << "BTestShell::UninstallPatches(): "
 551                         "Failed to acquire patch group lock!" << endl;
 552                 return;
 553         }
 554         if (fPatchGroup) {
 555                 fPatchGroup->Restore();
 556                 delete fPatchGroup;
 557                 fPatchGroup = NULL;
 558         }
 559 #endif // ! NO_ELF_SYMBOL_PATCHING
 560 }
 561
 562 #ifndef NO_ELF_SYMBOL_PATCHING
 563
 564 // _Debugger
 565 _EXPORT
 566 void
 567 BTestShell::_Debugger(const char *message)
 568 {
 569         if (!this || !fPatchGroup) {
 570                 debugger(message);
 571                 return;
 572         }
 573         BAutolock locker(fPatchGroupLocker);
 574         if (!locker.IsLocked() || !fPatchGroup) {
 575                 debugger(message);
 576                 return;
 577         }
 578 cout << "debugger() called: " << message << endl;
 579         void *var = tls_get(fTLSDebuggerCall);
 580         if (var)
 581                 tls_set(fTLSDebuggerCall, (void*)((int)var + 1));
 582         else
 583                 (*fOldDebuggerHook)(message);
 584 }
 585
 586 // _LoadAddOn
 587 _EXPORT
 588 image_id
 589 BTestShell::_LoadAddOn(const char *path)
 590 {
 591         if (!this || !fPatchGroup)
 592                 return load_add_on(path);
 593         BAutolock locker(fPatchGroupLocker);
 594         if (!locker.IsLocked() || !fPatchGroup)
 595                 return load_add_on(path);
 596         image_id result = (*fOldLoadAddOnHook)(path);
 597         fPatchGroup->Update();
 598         return result;
 599 }
 600
 601 // _UnloadAddOn
 602 _EXPORT
 603 status_t
 604 BTestShell::_UnloadAddOn(image_id image)
 605 {
 606         if (!this || !fPatchGroup)
 607                 return unload_add_on(image);
 608         BAutolock locker(fPatchGroupLocker);
 609         if (!locker.IsLocked() || !fPatchGroup)
 610                 return unload_add_on(image);
 611
 612         if (!this || !fPatchGroup)
 613                 return unload_add_on(image);
 614         status_t result = (*fOldUnloadAddOnHook)(image);
 615         fPatchGroup->Update();
 616         return result;
 617 }
 618
 619 // _DebuggerHook
 620 _EXPORT
 621 void
 622 BTestShell::_DebuggerHook(const char *message)
 623 {
 624         fGlobalShell->_Debugger(message);
 625 }
 626
 627 // _LoadAddOnHook
 628 _EXPORT
 629 image_id
 630 BTestShell::_LoadAddOnHook(const char *path)
 631 {
 632         return fGlobalShell->_LoadAddOn(path);
 633 }
 634
 635 // _UnloadAddOnHook
 636 _EXPORT
 637 status_t
 638 BTestShell::_UnloadAddOnHook(image_id image)
 639 {
 640         return fGlobalShell->_UnloadAddOn(image);
 641 }
 642
 643 #endif // ! NO_ELF_SYMBOL_PATCHING