kde3support/kunittest/runner.cpp

   1 /**
   2  *
   3  * Copyright (C)  2004  Zack Rusin <zack@kde.org>
   4  * Copyright (C)  2005  Jeroen Wijnhout <Jeroen.Wijnhout@kdemail.net>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright
  11  *   notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *   notice, this list of conditions and the following disclaimer in the
  14  *   documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26  */
  27
  28 #include "runner.h"
  29
  30 #include <stdio.h>
  31 #include <iostream>
  32 using namespace std;
  33
  34 #include <QtCore/QRegExp>
  35 #include <QtCore/QDir>
  36 #include <QtCore/QMetaEnum>
  37
  38 #include <kdebug.h>
  39 #include <klibloader.h>
  40 #include <kglobal.h>
  41 #include <kstandarddirs.h>
  42
  43 #include "tester.h"
  44
  45 namespace KUnitTest
  46 {
  47     Runner *Runner::s_self = 0L;
  48     bool    Runner::s_debugCapturingEnabled = false;
  49
  50     void Runner::registerTester(const char *name, Tester *test)
  51     {
  52         Runner::self()->m_registry.insert(name, test);
  53     }
  54
  55     void Runner::loadModules(const QString &folder, const QString &query)
  56     {
  57         QRegExp reQuery(query);
  58         QDir dir(folder, "kunittest_*.la");
  59
  60         // Add the folder to the "module" resource such that the KLibLoader can
  61         // find the modules in this folder.
  62         KGlobal::dirs()->addResourceDir("module", folder);
  63         kDebug() << "Looking in folder: " << dir.absolutePath();
  64
  65         // Get a list of all modules.
  66         QStringList modules = dir.entryList();
  67
  68         for ( int i = 0; i < modules.count(); ++i )
  69         {
  70             QString module = modules[i];
  71             kDebug() << "Module: " << dir.absolutePath() + '/' + module;
  72
  73             if ( reQuery.indexIn(module) != -1 )
  74             {
  75                 // strip the .la extension
  76                 module.truncate(module.length()-3);
  77                 KPluginLoader loader(module.toLocal8Bit());
  78                 KPluginFactory *factory = loader.factory();
  79                 if ( factory )
  80                     factory->create<QObject>();
  81                 else {
  82                     kWarning() << "\tError loading " << module << " : " << loader.errorString();
  83                     ::exit( 1 );
  84                 }
  85             }
  86             else
  87                 kDebug() << "\tModule doesn't match.";
  88         }
  89     }
  90
  91     void Runner::setDebugCapturingEnabled(bool enabled)
  92     {
  93       s_debugCapturingEnabled = enabled;
  94     }
  95
  96     //RegistryType &Runner::registry()
  97     Registry &Runner::registry()
  98     {
  99         return m_registry;
 100     }
 101
 102     int Runner::numberOfTestCases()
 103     {
 104         return m_registry.count();
 105     }
 106
 107     Runner *Runner::self()
 108     {
 109         if ( !s_self )
 110             s_self = new Runner();
 111
 112         return s_self;
 113     }
 114
 115
 116     Runner::Runner()
 117     {
 118         reset();
 119     }
 120
 121     int Runner::numberOfTests() const
 122     {
 123         return globalSteps;
 124     }
 125
 126     int Runner::numberOfPassedTests() const
 127     {
 128         return globalPasses;
 129     }
 130
 131     int Runner::numberOfFailedTests() const
 132     {
 133         return globalFails;
 134     }
 135
 136     int Runner::numberOfExpectedFailures() const
 137     {
 138         return globalXFails;
 139     }
 140
 141     int Runner::numberOfSkippedTests() const
 142     {
 143         return globalSkipped;
 144     }
 145
 146     void Runner::reset()
 147     {
 148         globalSteps = 0;
 149         globalPasses = 0;
 150         globalFails = 0;
 151         globalXFails = 0;
 152         globalXPasses = 0;
 153         globalSkipped = 0;
 154     }
 155
 156     int Runner::runTests()
 157     {
 158         globalSteps = 0;
 159         globalPasses = 0;
 160         globalFails = 0;
 161         globalXFails = 0;
 162         globalXPasses = 0;
 163         globalSkipped = 0;
 164
 165         cout << "# Running normal tests... #" << endl << endl;
 166
 167         Registry::const_iterator it = m_registry.constBegin();
 168         for( ; it != m_registry.constEnd(); ++it )
 169             runTest( it.key( ) );
 170
 171 #if 0 // very thorough, but not very readable
 172         cout << "# Done with normal tests:" << endl;
 173         cout << "  Total test cases: " << m_registry.count() << endl;
 174         cout << "  Total test steps                                 : " << globalSteps << endl;
 175         cout << "    Total passed test steps (including unexpected) : " << globalPasses << endl;
 176         cout << "      Total unexpected passed test steps           :  " << globalXPasses << endl;
 177         cout << "    Total failed test steps (including expected)   :  " << globalFails << endl;
 178         cout << "      Total expected failed test steps             :  " << globalXFails << endl;
 179         cout << "    Total skipped test steps                       :  " << globalSkipped << endl;
 180 #else
 181         unsigned int numTests = m_registry.count(); // should this be globalSteps instead?
 182         QString str;
 183         if ( globalFails == 0 )
 184             if ( globalXFails == 0 )
 185                 str = QString( "All %1 tests passed" ).arg( numTests );
 186             else
 187                 str = QString( "All %1 tests behaved as expected (%2 expected failures)" ).arg( numTests ).arg( globalXFails );
 188         else
 189             if ( globalXPasses == 0 )
 190                 str = QString( "%1 of %2 tests failed" ).arg( globalFails ).arg( numTests );
 191             else
 192                 str = QString( "%1 of %2 tests did not behave as expected (%1 unexpected passes)" ).arg( globalFails ).arg( numTests ).arg( globalXPasses );
 193         if ( globalSkipped )
 194             str += QString( " (%1 tests skipped)" ).arg( globalSkipped );
 195         cout << str.toLocal8Bit().constData() << endl;
 196 #endif
 197
 198         return m_registry.count();
 199     }
 200
 201     void Runner::runMatchingTests(const QString &prefix)
 202     {
 203         Registry::const_iterator it = m_registry.constBegin();
 204         for( ; it != m_registry.constEnd(); ++it )
 205             if ( QString( it.key() ).startsWith(prefix) )
 206                 runTest( it.key() );
 207     }
 208
 209     void Runner::runTest(const char *name)
 210     {
 211         Tester *test = m_registry.value( name );
 212         if ( !test ) return;
 213
 214         if ( s_debugCapturingEnabled )
 215         {
 216           cout << "KUnitTest_Debug_Start[" << name << "]" << endl;
 217         }
 218
 219         test->results()->clear();
 220         test->allTests();
 221
 222         if ( s_debugCapturingEnabled )
 223         {
 224           cout << "KUnitTest_Debug_End[" << name << "]" << endl << endl << flush;
 225         }
 226
 227         int numPass = 0;
 228         int numFail = 0;
 229         int numXFail = 0;
 230         int numXPass = 0;
 231         int numSkip = 0;
 232
 233         if ( test->inherits("KUnitTest::SlotTester") )
 234         {
 235             SlotTester *sltest = static_cast<SlotTester*>(test);
 236             foreach( TestResults* res, sltest->resultsList() )
 237             {
 238                 numPass += res->passed() + res->xpasses();
 239                 numFail += res->errors() + res->xfails();
 240                 numXFail += res->xfails();
 241                 numXPass += res->xpasses();
 242                 numSkip += res->skipped();
 243                 globalSteps += res->testsFinished();
 244             }
 245         }
 246         else
 247         {
 248             numPass= test->results()->passed() + test->results()->xpasses();
 249             numFail= test->results()->errors() + test->results()->xfails();
 250             numXFail = test->results()->xfails();
 251             numXPass = test->results()->xpasses();
 252             numSkip= test->results()->skipped();
 253             globalSteps += test->results()->testsFinished();
 254         }
 255
 256
 257         globalPasses += numPass;
 258         globalFails += numFail;
 259         globalXFails += numXFail;
 260         globalXPasses += numXPass;
 261         globalSkipped += numSkip;
 262
 263         cout << name << " - ";
 264         cout << numPass << " test" << ( ( 1 == numPass )?"":"s") << " passed";
 265         if ( 0 < test->results()->xpassList().count() ) {
 266             cout << " (" << numXPass << " unexpected pass" << ( ( 1 == numXPass )?"":"es") << ")";
 267         }
 268         cout << ", " << numFail << " test" << ( ( 1 == numFail )?"":"s") << " failed";
 269         if ( 0 < numXFail  ) {
 270             cout << " (" << numXFail << " expected failure" << ( ( 1 == numXFail )?"":"s") << ")";
 271         }
 272         if ( 0 < numSkip ) {
 273             cout << "; also " << numSkip << " skipped";
 274         }
 275         cout  << endl;
 276
 277         if ( 0 < numXPass  ) {
 278         cout << "    Unexpected pass" << ( ( 1 == numXPass )?"":"es") << ":" << endl;
 279         QStringList list = test->results()->xpassList();
 280         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
 281             cout << "\t" << (*itr).toLatin1().constData() << endl;
 282         }
 283         }
 284         if ( 0 < (numFail - numXFail) ) {
 285         cout << "    Unexpected failure" << ( ( 1 == numFail )?"":"s") << ":" << endl;
 286         QStringList list = test->results()->errorList();
 287         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
 288             cout << "\t" << (*itr).toLatin1().constData() << endl;
 289         }
 290         }
 291         if ( 0 < numXFail ) {
 292         cout << "    Expected failure" << ( ( 1 == numXFail)?"":"s") << ":" << endl;
 293         QStringList list = test->results()->xfailList();
 294         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
 295             cout << "\t" << (*itr).toLatin1().constData() << endl;
 296         }
 297         }
 298         if ( 0 < numSkip ) {
 299             cout << "    Skipped test" << ( ( 1 == numSkip )?"":"s") << ":" << endl;
 300             QStringList list = test->results()->skipList();
 301             for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
 302             cout << "\t" << (*itr).toLatin1().constData() << endl;
 303             }
 304         }
 305         cout << endl;
 306
 307         emit finished(name, test);
 308     }
 309 }
 310
 311 #include "runner.moc"
 312