fix tricky regression noticed by Vyacheslav Tokarev on Google Reader.
[kdelibs.git] / khtml / test_regression_gui_window.cpp
blob3c19f28e81f354827289f5e935168198b7c6f8a1
1 /**
2 * This file is part of the KDE project
4 * Copyright (C) 2006 Nikolas Zimmermann <zimmermann@kde.org>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public License
17 * along with this library; see the file COPYING.LIB. If not, write to
18 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
23 #include <assert.h>
24 #include <signal.h>
26 #include <QtCore/QFile>
27 #include <QtCore/QTimer>
28 #include <QtCore/QProcess>
29 #include <QtCore/QFileInfo>
30 #include <QtCore/QTextStream>
31 #include <QtGui/QMainWindow>
33 #include <kiconloader.h>
34 #include <kmessagebox.h>
35 #include <kconfig.h>
36 #include <kfiledialog.h>
38 #include "test_regression_gui_window.moc"
40 // Taken from QUrl
41 #define Q_HAS_FLAG(a, b) ( ((a) & (b)) == (b) )
42 #define Q_SET_FLAG(a, b) { (a) |= (b); }
43 #define Q_UNSET_FLAG(a, b) { (a) &= ~(b); }
45 TestRegressionWindow::TestRegressionWindow(QWidget *parent)
46 : QMainWindow(parent), m_flags(None), m_runCounter(0), m_testCounter(0), m_totalTests(0),
47 m_totalTestsJS(0), m_totalTestsDOMTS(0), m_lastResult(Unknown),
48 m_browserPart(0), m_activeProcess(0), m_activeTreeItem(0),
49 m_suspended(false), m_justProcessingQueue(false)
51 m_ui.setupUi(this);
53 // Setup actions/connections
54 connect(m_ui.actionOnly_run_JS_tests, SIGNAL(toggled(bool)), SLOT(toggleJSTests(bool)));
55 connect(m_ui.actionOnly_run_HTML_tests, SIGNAL(toggled(bool)), SLOT(toggleHTMLTests(bool)));
56 connect(m_ui.actionDo_not_suppress_debug_output, SIGNAL(toggled(bool)), SLOT(toggleDebugOutput(bool)));
57 connect(m_ui.actionDo_not_use_Xvfb, SIGNAL(toggled(bool)), SLOT(toggleNoXvfbUse(bool)));
58 connect(m_ui.actionSpecify_tests_directory, SIGNAL(triggered(bool)), SLOT(setTestsDirectory()));
59 connect(m_ui.actionSpecify_khtml_directory, SIGNAL(triggered(bool)), SLOT(setKHTMLDirectory()));
60 connect(m_ui.actionSpecify_output_directory, SIGNAL(triggered(bool)), SLOT(setOutputDirectory()));
61 connect(m_ui.actionRun_tests, SIGNAL(triggered(bool)), SLOT(runTests()));
63 connect(m_ui.pauseContinueButton, SIGNAL(clicked(bool)), SLOT(pauseContinueButtonClicked()));
64 connect(m_ui.saveLogButton, SIGNAL(clicked(bool)), SLOT(saveLogButtonClicked()));
66 connect(m_ui.treeWidget, SIGNAL(customContextMenuRequested(const QPoint &)),
67 this, SLOT(treeWidgetContextMenuRequested(const QPoint &)));
69 // Setup actions' default state
70 m_ui.progressBar->setValue(0);
71 m_ui.textEdit->setReadOnly(true);
72 m_ui.actionRun_tests->setEnabled(false);
73 m_ui.pauseContinueButton->setEnabled(false);
75 m_ui.treeWidget->headerItem()->setTextAlignment(0, Qt::AlignLeft);
76 m_ui.treeWidget->headerItem()->setText(0, i18n("Available Tests: 0"));
78 // Load default values for tests directory/khtml directory...
79 KConfig config("testregressiongui", KConfig::SimpleConfig);
80 KConfigGroup grp = config.group("<default>");
82 m_testsUrl = KUrl::fromPath(grp.readPathEntry("TestsDirectory", QString()));
83 m_khtmlUrl = KUrl::fromPath(grp.readPathEntry("KHTMLDirectory", QString()));
85 initTestsDirectory();
87 // Init early visible items in the text edit...
88 initLegend();
89 initOutputBrowser();
92 TestRegressionWindow::~TestRegressionWindow()
94 if(m_activeProcess)
96 m_activeProcess->kill();
98 /* This leads to:
99 * QProcess object destroyed while process is still running.
100 * Any idea why??
101 delete m_activeProcess;
104 m_activeProcess = 0;
108 void TestRegressionWindow::toggleJSTests(bool checked)
110 if(checked)
112 Q_SET_FLAG(m_flags, JSTests)
113 Q_UNSET_FLAG(m_flags, HTMLTests)
115 m_ui.actionOnly_run_HTML_tests->setChecked(false);
117 else
118 Q_UNSET_FLAG(m_flags, JSTests)
120 // Eventually update progress bar range...
121 updateProgressBarRange();
124 void TestRegressionWindow::toggleHTMLTests(bool checked)
126 if(checked)
128 Q_SET_FLAG(m_flags, HTMLTests)
129 Q_UNSET_FLAG(m_flags, JSTests)
131 m_ui.actionOnly_run_JS_tests->setChecked(false);
133 else
134 Q_UNSET_FLAG(m_flags, HTMLTests)
136 // Eventually update progress bar range...
137 updateProgressBarRange();
140 void TestRegressionWindow::toggleDebugOutput(bool checked)
142 if(checked)
143 Q_SET_FLAG(m_flags, DebugOutput)
144 else
145 Q_UNSET_FLAG(m_flags, DebugOutput)
148 void TestRegressionWindow::toggleNoXvfbUse(bool checked)
150 if(checked)
151 Q_SET_FLAG(m_flags, NoXvfbUse)
152 else
153 Q_UNSET_FLAG(m_flags, NoXvfbUse)
156 void TestRegressionWindow::setTestsDirectory()
158 m_testsUrl = KFileDialog::getExistingDirectory();
160 initTestsDirectory();
161 loadOutputHTML();
164 void TestRegressionWindow::setOutputDirectory()
166 m_outputUrl = KFileDialog::getExistingDirectory();
167 loadOutputHTML();
170 void TestRegressionWindow::initTestsDirectory()
172 bool okay = !m_testsUrl.isEmpty();
173 if(okay)
175 const char *subdirs[] = { "tests", "baseline", "output", "resources" };
176 for(int i = 0; i <= 3; i++)
178 QFileInfo sourceDir(m_testsUrl.path() + "/" + subdirs[i]); //krazy:exclude=duoblequote_chars DOM demands chars
179 if(!sourceDir.exists() || !sourceDir.isDir())
181 KMessageBox::error(0, i18n("Please choose a valid 'khtmltests/regression/' directory."));
183 okay = false;
184 m_testsUrl = KUrl();
185 break;
190 if(okay)
192 // Clean up...
193 m_itemMap.clear();
194 m_ignoreMap.clear();
195 m_failureMap.clear();
196 m_directoryMap.clear();
198 m_ui.treeWidget->clear();
200 if(!m_khtmlUrl.isEmpty())
201 m_ui.actionRun_tests->setEnabled(true);
203 // Initialize map (to prevent assert below)...
204 m_directoryMap.insert(QString(), QStringList());
206 // Setup root tree widget item...
207 (void) new QTreeWidgetItem(m_ui.treeWidget, QStringList(m_testsUrl.path() + "/tests"));
209 // Check for ignore & failure file in root directory...
210 QString ignoreFile = m_testsUrl.path() + "/tests/ignore";
211 QString failureFile = m_testsUrl.path() + "/tests/KNOWN_FAILURES";
213 QStringList ignoreFileList = readListFile(ignoreFile);
214 QStringList failureFileList = readListFile(failureFile);
216 if(!ignoreFileList.isEmpty())
217 m_ignoreMap.insert(QString(), ignoreFileList);
219 if(!failureFileList.isEmpty())
220 m_failureMap.insert(QString(), failureFileList);
222 // Remember directory...
223 KConfig config("testregressiongui", KConfig::SimpleConfig);
224 KConfigGroup grp = config.group("<default>");
225 grp.writePathEntry("TestsDirectory", m_testsUrl.path());
227 // Start listing directory...
228 KUrl listUrl = m_testsUrl; listUrl.addPath("tests");
229 KIO::ListJob *job = KIO::listRecursive(listUrl, KIO::HideProgressInfo, false /* no hidden files */);
231 connect(job, SIGNAL(result(KJob *)), SLOT(directoryListingFinished(KJob *)));
233 connect(job, SIGNAL(entries(KIO::Job *, const KIO::UDSEntryList &)),
234 this, SLOT(directoryListingResult(KIO::Job *, const KIO::UDSEntryList &)));
238 void TestRegressionWindow::setKHTMLDirectory()
240 m_khtmlUrl = KFileDialog::getExistingDirectory();
242 if(!m_khtmlUrl.isEmpty())
244 const char *subdirs[] = { "css", "dom", "xml", "html" }; // That's enough ;-)
245 for(int i = 0; i <= 3; i++)
247 QFileInfo sourceDir(m_khtmlUrl.path() + "/" + subdirs[i]); //krazy:exclude=duoblequote_chars DOM demands chars
248 if(!sourceDir.exists() || !sourceDir.isDir())
250 KMessageBox::error(0, i18n("Please choose a valid 'khtml/' build directory."));
252 m_khtmlUrl = KUrl();
253 break;
257 // Remember directory...
258 KConfig config("testregressiongui", KConfig::SimpleConfig);
259 KConfigGroup grp = config.group("<default>");
260 grp.writePathEntry("KHTMLDirectory", m_khtmlUrl.path());
262 if(!m_testsUrl.isEmpty() && !m_khtmlUrl.isEmpty())
263 m_ui.actionRun_tests->setEnabled(true);
267 void TestRegressionWindow::directoryListingResult(KIO::Job *, const KIO::UDSEntryList &list)
269 KIO::UDSEntryList::ConstIterator it = list.constBegin();
270 const KIO::UDSEntryList::ConstIterator end = list.constEnd();
272 for(; it != end; ++it)
274 const KIO::UDSEntry &entry = *it;
276 QString name = entry.stringValue(KIO::UDSEntry::UDS_NAME);
277 if(entry.isDir()) // Create new map entry...
279 assert(m_directoryMap.constFind(name) == m_directoryMap.constEnd());
280 m_directoryMap.insert(name, QStringList());
282 QString ignoreFile = m_testsUrl.path() + "/tests/" + name + "/ignore";
283 QString failureFile = m_testsUrl.path() + "/tests/" + name + "/KNOWN_FAILURES";
285 QStringList ignoreFileList = readListFile(ignoreFile);
286 QStringList failureFileList = readListFile(failureFile);
288 if(!ignoreFileList.isEmpty())
289 m_ignoreMap.insert(name, ignoreFileList);
291 if(!failureFileList.isEmpty())
292 m_failureMap.insert(name, failureFileList);
294 else if(name.endsWith(".html") || name.endsWith(".htm") ||
295 name.endsWith(".xhtml") || name.endsWith(".xml") || name.endsWith(".js"))
297 int lastSlashPos = name.lastIndexOf('/');
299 QString cachedDirectory = (lastSlashPos > 0 ? name.mid(0, lastSlashPos) : QString());
300 QString cachedFilename = name.mid(lastSlashPos + 1);
302 assert(m_directoryMap.constFind(cachedDirectory) != m_directoryMap.constEnd());
303 m_directoryMap[cachedDirectory].append(cachedFilename);
308 void TestRegressionWindow::directoryListingFinished(KJob *)
310 QTreeWidgetItem *topLevelItem = m_ui.treeWidget->topLevelItem(0);
312 // Gather a lot of statistics...
313 unsigned long availableDomFiles = 0;
314 unsigned long availableDumpFiles = 0;
315 unsigned long availableRenderFiles = 0;
317 unsigned long ignoredJSTests = 0;
318 unsigned long availableJSTests = 0;
320 unsigned long ignoredXMLTests = 0;
321 unsigned long availableXMLTests = 0;
323 unsigned long ignoredHTMLTests = 0;
324 unsigned long availableHTMLTests = 0;
326 unsigned long ignoredDOMTSTests = 0;
327 unsigned long availableDOMTSTests = 0;
329 // Start the actual data processing...
330 QMap<QString, QStringList>::const_iterator it = m_directoryMap.constBegin();
331 const QMap<QString, QStringList>::const_iterator end = m_directoryMap.constEnd();
333 for(; it != end; ++it)
335 QString directory = it.key();
336 QStringList filenames = it.value();
338 if(filenames.isEmpty()) // Do not add empty directories at all...
339 continue;
341 bool hasIgnores = (m_ignoreMap.constFind(directory) != m_directoryMap.constEnd());
342 bool hasFailures = (m_failureMap.constFind(directory) != m_failureMap.constEnd());
344 // Extract parent directory...
345 int position = directory.lastIndexOf('/');
347 QString parentDirectory = directory.mid(0, (position == -1 ? 0 : position));
348 QString parentDirectoryItem = directory.mid(position + 1);
350 bool hasParentIgnores = (m_ignoreMap.constFind(parentDirectory) != m_directoryMap.constEnd());
351 bool hasParentFailures = (m_failureMap.constFind(parentDirectory) != m_failureMap.constEnd());
353 // Sort in ascending order...
354 filenames.sort();
356 QStringList::const_iterator it2 = filenames.constBegin();
357 const QStringList::const_iterator end2 = filenames.constEnd();
359 // Create new tree widget item for the active directory...
360 QTreeWidgetItem *parent = topLevelItem;
362 if(!directory.isEmpty())
364 parent = new QTreeWidgetItem(topLevelItem, QStringList(directory));
366 // Directory is completely ignored, mark it 'yellow'...
367 if(hasParentIgnores && m_ignoreMap[parentDirectory].contains(parentDirectoryItem))
368 parent->setIcon(0, m_ignorePixmap);
370 // Directory is completely known to fail, mark it 'red'...
371 if(hasParentFailures && m_failureMap[parentDirectory].contains(parentDirectoryItem))
372 parent->setIcon(0, m_failKnownPixmap);
375 // Add all contained files as new items below 'parent'...
376 for(; it2 != end2; ++it2)
378 QString test = (*it2);
379 QString cacheName = directory + "/" + test; //krazy:exclude=duoblequote_chars DOM demands chars
381 QTreeWidgetItem *testItem = new QTreeWidgetItem(parent, QStringList(KUrl(test).path()));
383 // Remember name <-> item pair...
384 assert(m_itemMap.contains(cacheName));
385 m_itemMap.insert(cacheName, testItem);
387 bool ignore = (hasIgnores && m_ignoreMap[directory].contains(test));
388 bool ignoreParent = (hasParentIgnores && m_ignoreMap[parentDirectory].contains(parentDirectoryItem));
390 bool failure = (hasFailures && m_failureMap[directory].contains(test));
392 // Check baseline directory for this test...
393 QString baseLinePath = m_testsUrl.path() + "/baseline/" + cacheName;
395 bool dom[9], render[9];
396 for(unsigned int i = 0; i < 9; ++i)
398 if(i == 0)
400 dom[i] = (QFileInfo(baseLinePath + "-dom").exists());
401 render[i] = (QFileInfo(baseLinePath + "-render").exists());
403 else
405 dom[i] = (QFileInfo(baseLinePath + "-" + QString::number(i) + "-dom").exists()); //krazy:exclude=duoblequote_chars DOM demands chars
406 render[i] = (QFileInfo(baseLinePath + "-" + QString::number(i) + "-render").exists()); //krazy:exclude=duoblequote_chars DOM demands chars
410 bool dump = (QFileInfo(baseLinePath + "-dump.png").exists());
412 // Ignored tests are marked 'yellow'...
413 if(ignore)
414 testItem->setIcon(0, m_ignorePixmap);
416 // Tests, known to fail, are marked 'red'...
417 if(failure)
418 testItem->setIcon(0, m_failKnownPixmap);
420 // Detect whether the tests has no corresponding baseline items...
421 if(!ignore && !failure)
423 if(!dom[0] && !dump && !render && !cacheName.endsWith(".js") && !cacheName.startsWith("domts"))
425 // See if parent directory is completely ignored...
426 if(!ignoreParent)
427 testItem->setIcon(0, m_noBaselinePixmap);
431 // Update statistics...
432 if(dump)
433 availableDumpFiles++;
435 for(unsigned i = 0; i < 9; ++i)
437 if(dom[i])
438 availableDomFiles++;
440 if(render[i])
441 availableRenderFiles++;
444 // Count DOM Testsuite files separated... (these have no baseline items!)
445 if(cacheName.startsWith("domts"))
447 // See if parent directory is completely ignored...
448 if(ignore || ignoreParent)
449 ignoredDOMTSTests++;
450 else
451 availableDOMTSTests++;
454 if(cacheName.endsWith(".html") || cacheName.endsWith(".htm") || cacheName.endsWith(".xhtml"))
456 if(ignore || ignoreParent)
457 ignoredHTMLTests++;
458 else
459 availableHTMLTests++;
461 else if(cacheName.endsWith(".xml"))
463 if(ignore || ignoreParent)
464 ignoredXMLTests++;
465 else
466 availableXMLTests++;
468 else if(cacheName.endsWith(".js"))
470 unsigned long containedTests = 0;
472 // Try hard to _ESTIMATE_ the number of tests...
473 // I really meant estimate, no way to calculate it perfectly.
475 QString jsFilePath = m_testsUrl.path() + "/tests/" + cacheName;
476 assert(QFileInfo(jsFilePath).exists() == true);
478 QStringList fileList = readListFile(jsFilePath);
479 QString fileContent = fileList.join("");
481 // #1 -> Check js file for the 'reportResult' calls...
482 containedTests = fileContent.count("reportResult");
484 // #2 -> Check js file for 'openPage' calls...
485 containedTests += fileContent.count("openPage");
487 // #3 -> Check js file for 'checkOutput' calls...
488 containedTests += fileContent.count("checkOutput");
490 // #4 -> Fallback for ie. mozilla/ecma files...
491 if(containedTests == 0) // Doesn't use 'reportResult' scheme...
492 containedTests++;
494 if(ignore || ignoreParent)
495 ignoredJSTests += containedTests;
496 else
497 availableJSTests += containedTests;
502 // Now we can calculate all ignored/available tests...
503 unsigned long ignoredTests = ignoredJSTests + ignoredXMLTests + ignoredHTMLTests;
504 unsigned long availableTests = availableJSTests + availableXMLTests + availableHTMLTests;
506 // This estimates the number of total tests, depending on the mode...
507 m_totalTests = availableDomFiles + availableDumpFiles + availableRenderFiles +
508 availableDOMTSTests + availableJSTests;
510 m_totalTestsJS = availableJSTests;
511 m_totalTestsDOMTS = availableDOMTSTests;
513 // Update progress bar range...
514 updateProgressBarRange();
516 QString statistics = QString("<body><table border='0' align='center' cellspacing='15'>") +
517 QString("<tr valign='top'><td colspan='3'><center><b>Statistics</b></center></td></tr>") +
518 QString("<tr valign='middle'><td>JS Tests</td><td>" + QString::number(availableJSTests) + "</td><td>(" + QString::number(ignoredJSTests) + " ignored)</td></tr>") +
519 QString("<tr valign='middle'><td>XML Tests</td><td>" + QString::number(availableXMLTests) + "</td><td>(" + QString::number(ignoredXMLTests) + " ignored)</td></tr>") +
520 QString("<tr valign='middle'><td>HTML Tests</td><td>" + QString::number(availableHTMLTests) + "</td><td>(" + QString::number(ignoredHTMLTests) + " ignored)</td></tr>") +
521 QString("</table></body>");
523 // Go to end...
524 QTextCursor cursor = m_ui.textEdit->textCursor();
525 cursor.movePosition(QTextCursor::End);
526 m_ui.textEdit->setTextCursor(cursor);
528 // Insert statistics...
529 m_ui.textEdit->insertHtml(statistics);
531 // Update treeview...
532 m_ui.treeWidget->headerItem()->setText(0, i18n("Available Tests: %1 (ignored: %2)", availableTests, ignoredTests));
535 void TestRegressionWindow::updateProgressBarRange() const
537 if(m_totalTests != 0 && m_totalTestsJS != 0)
539 unsigned long totalTests = m_totalTests;
541 if(Q_HAS_FLAG(m_flags, JSTests))
542 totalTests = m_totalTestsJS;
543 else if(Q_HAS_FLAG(m_flags, HTMLTests))
545 totalTests -= m_totalTestsJS;
546 totalTests -= m_totalTestsDOMTS;
549 m_ui.progressBar->setRange(0, totalTests);
553 void TestRegressionWindow::pauseContinueButtonClicked()
555 assert(m_activeProcess != 0);
557 if(!m_suspended)
559 // Suspend process
560 kill(m_activeProcess->pid(), SIGSTOP);
562 m_suspended = true;
563 m_ui.pauseContinueButton->setText(i18n("Continue"));
565 else
567 // Continue process
568 kill(m_activeProcess->pid(), SIGCONT);
570 m_suspended = false;
571 m_ui.pauseContinueButton->setText(i18n("Pause"));
575 void TestRegressionWindow::saveLogButtonClicked()
577 assert(m_activeProcess == 0);
578 m_saveLogUrl = KFileDialog::getExistingDirectory();
580 QString fileName = m_saveLogUrl.path() + "/logOutput.html";
581 if(QFileInfo(fileName).exists())
583 // Remove file if already existent...
584 QFile file(fileName);
585 if(!file.remove())
587 kError() << " Can't remove " << fileName << endl;
588 exit(1);
593 void TestRegressionWindow::runTests()
595 // Run in all-in-one mode...
596 m_runCounter = 0;
597 m_testCounter = 0;
599 initRegressionTesting(QString());
602 void TestRegressionWindow::runSingleTest()
604 assert(m_activeTreeItem != 0);
606 QString testFileName = pathFromItem(m_activeTreeItem);
608 // Run in single-test mode...
609 m_runCounter = 0;
610 m_testCounter = -1;
612 initRegressionTesting(testFileName);
615 void TestRegressionWindow::initRegressionTesting(const QString &testFileName)
617 assert(m_activeProcess == 0);
619 m_activeProcess = new QProcess();
620 m_activeProcess->setReadChannelMode(QProcess::MergedChannels);
622 QStringList environment = QProcess::systemEnvironment();
623 environment << "KDE_DEBUG=false"; // No Dr. Konqi please!
625 QString program = m_khtmlUrl.path() + "/.libs/testregression";
626 QString program2 = m_khtmlUrl.path() + "/testregression"; // with CMake, it's in $buildir/bin
628 if(!QFileInfo(program).exists())
630 if(!QFileInfo(program2).exists())
632 KMessageBox::error(0, i18n("Cannot find testregression executable."));
633 return;
635 else
637 program = program2;
641 QStringList arguments;
642 arguments << "--base" << m_testsUrl.path();
644 if(!m_outputUrl.isEmpty())
645 arguments << "--output" << m_outputUrl.path();
646 if(!testFileName.isEmpty())
647 arguments << "--test" << testFileName;
649 if(Q_HAS_FLAG(m_flags, JSTests))
650 arguments << "--js";
651 if(Q_HAS_FLAG(m_flags, HTMLTests))
652 arguments << "--html";
653 if(Q_HAS_FLAG(m_flags, DebugOutput))
654 arguments << "--debug";
655 if(Q_HAS_FLAG(m_flags, NoXvfbUse))
656 arguments << "--noxvfb";
658 connect(m_activeProcess, SIGNAL(finished(int, QProcess::ExitStatus)), SLOT(testerExited(int, QProcess::ExitStatus)));
659 connect(m_activeProcess, SIGNAL(readyReadStandardOutput()), SLOT(testerReceivedData()));
661 // Clear processing queue before starting...
662 m_processingQueue.clear();
664 // Clean up gui...
665 m_ui.textEdit->clear();
666 m_ui.progressBar->reset();
667 m_ui.saveLogButton->setEnabled(false);
668 m_ui.actionRun_tests->setEnabled(false);
669 m_ui.pauseContinueButton->setEnabled(true);
671 // Start regression testing process...
672 m_activeProcess->setEnvironment(environment);
673 m_activeProcess->start(program, arguments, QIODevice::ReadOnly);
676 void TestRegressionWindow::initOutputBrowser()
678 assert(m_browserPart == 0);
679 m_browserPart = new KHTMLPart(m_ui.secondTab, m_ui.secondTab, KHTMLPart::BrowserViewGUI);
681 // Setup vertical layout for the browser widget...
682 QVBoxLayout *layout = new QVBoxLayout();
683 layout->addWidget(m_browserPart->widget());
684 m_ui.secondTab->setLayout(layout);
686 m_browserPart->setJavaEnabled(true);
687 m_browserPart->setJScriptEnabled(true);
688 m_browserPart->setPluginsEnabled(true);
689 m_browserPart->setURLCursor(QCursor(Qt::PointingHandCursor));
691 m_browserPart->widget()->show();
693 // Check if there is already an output/index.html present...
694 loadOutputHTML();
697 void TestRegressionWindow::loadOutputHTML() const
699 if(m_testsUrl.isEmpty())
700 return;
702 QString fileName = m_testsUrl.path() + "/output/index.html";
703 if(!m_outputUrl.isEmpty())
704 fileName = m_outputUrl.path() + "/index.html";
706 QFileInfo indexHtml(fileName);
707 if(indexHtml.exists())
709 m_browserPart->openUrl(KUrl::fromPath(fileName));
710 m_ui.tabWidget->setTabEnabled(1, true);
712 else
713 m_ui.tabWidget->setTabEnabled(1, false);
716 void TestRegressionWindow::updateItemStatus(TestResult result, QTreeWidgetItem *item, const QString &testFileName)
718 if(!item)
719 return;
721 // Ensure item is visible...
722 QTreeWidgetItem *parent = item;
723 while(parent != 0)
725 m_ui.treeWidget->setItemExpanded(parent, true);
726 parent = parent->parent();
729 m_ui.treeWidget->scrollToItem(item);
731 bool updateIcon = true;
732 if(m_lastName == testFileName && !m_lastName.isEmpty())
734 if(m_lastResult == result)
735 updateIcon = false;
736 else if((m_lastResult == Pass || m_lastResult == PassUnexpected) &&
737 (result == Fail || result == FailKnown || result == Crash))
739 // If one part of the test (render/dom/paint) passed,
740 // and the current part fails, update to 'failed' icon...
741 updateIcon = true;
743 else if((m_lastResult == Fail || m_lastResult == FailKnown || m_lastResult == Crash) &&
744 (result == Pass || result == PassUnexpected))
746 // If one part of the test (render/dom/paint) failed,
747 // and the current part passes, don't update to 'passed' icon...
748 updateIcon = false;
752 // Update icon, if necessary...
753 if(updateIcon)
755 if(result == Crash)
756 item->setIcon(0, m_crashPixmap);
757 else if(result == Fail)
758 item->setIcon(0, m_failPixmap);
759 else if(result == FailKnown)
760 item->setIcon(0, m_failKnownPixmap);
761 else if(result == Pass)
762 item->setIcon(0, m_passPixmap);
763 else if(result == PassUnexpected)
764 item->setIcon(0, m_passUnexpectedPixmap);
765 else // Unhandled state...
766 assert(false);
769 // Remember test & result...
770 m_lastResult = result;
771 m_lastName = testFileName;
772 m_activeTreeItem = item;
775 void TestRegressionWindow::initLegend()
777 // Init pixmaps...
778 m_failPixmap = QPixmap(":/test/pics/fail.xpm");
779 m_failKnownPixmap = QPixmap(":/test/pics/failKnown.xpm");
780 m_passPixmap = QPixmap(":/test/pics/pass.xpm");
781 m_passUnexpectedPixmap = QPixmap(":/test/pics/passUnexpected.xpm");
782 m_ignorePixmap = QPixmap(":/test/pics/ignore.xpm");
783 m_crashPixmap = QPixmap(":/test/pics/crash.xpm");
784 m_noBaselinePixmap = QPixmap(":/test/pics/noBaseline.xpm");
786 QString legend = QLatin1String("<body><center><font size='8'>Welcome to the khtml<br/>") +
787 QLatin1String("regression testing tool!</font></center><br/><br/>") +
788 QLatin1String("<table border='0' align='center' cellspacing='15'>") +
789 QLatin1String("<tr valign='top'><td colspan='2'><center><b>Legend</b></center></td></tr>") +
790 QLatin1String("<tr valign='middle'><td>Pass</td><td><img src=':/test/pics/pass.xpm'></td></tr>") +
791 QLatin1String("<tr valign='middle'><td>Pass unexpected</td><td><img src=':/test/pics/passUnexpected.xpm'></td></tr>") +
792 QLatin1String("<tr valign='middle'><td>Fail</td><td><img src=':/test/pics/fail.xpm'></td></tr>") +
793 QLatin1String("<tr valign='middle'><td>Fail known</td><td><img src=':/test/pics/failKnown.xpm'></td></tr>") +
794 QLatin1String("<tr valign='middle'><td>Ignore</td><td><img src=':/test/pics/ignore.xpm'></td></tr>") +
795 QLatin1String("<tr valign='middle'><td>Baseline missing</td><td><img src=':/test/pics/noBaseline.xpm'></td></tr>") +
796 QLatin1String("<tr valign='middle'><td>Crash</td><td><img src=':/test/pics/crash.xpm'></td></tr>") +
797 QLatin1String("</table></body>");
799 m_ui.textEdit->setHtml(legend);
802 void TestRegressionWindow::testerExited(int /* exitCode */, QProcess::ExitStatus exitStatus)
804 assert(m_activeProcess != 0);
805 assert(m_activeTreeItem != 0);
807 if(exitStatus == QProcess::CrashExit) // Special case: crash!
809 QTreeWidgetItem *useItem = m_activeTreeItem;
811 if(m_testCounter >= 0 || m_runCounter > 0) // Single-tests mode invoked on a directory OR All-test-mode
813 QTreeWidgetItem *parent = useItem->parent();
814 assert(parent != 0);
816 useItem = parent->child(parent->indexOfChild(useItem) + 1);
817 assert(useItem != 0);
820 // Reflect crashed test...
821 updateItemStatus(Crash, useItem, QString());
824 if(m_testCounter >= 0) // All-tests mode
825 m_ui.progressBar->setValue(m_ui.progressBar->maximum());
827 // Eventually save log output...
828 if(!m_saveLogUrl.isEmpty())
830 // We should close our written log with </body></html>.
831 m_processingQueue.enqueue(QString::fromLatin1("\n</body>\n</html>"));
833 if(!m_justProcessingQueue)
835 m_justProcessingQueue = true;
836 QTimer::singleShot(50, this, SLOT(processQueue()));
840 // Cleanup gui...
841 m_ui.saveLogButton->setEnabled(true);
842 m_ui.actionRun_tests->setEnabled(true);
843 m_ui.pauseContinueButton->setEnabled(false);
845 // Check if there is already an output/index.html present...
846 loadOutputHTML();
848 // Cleanup data..
849 delete m_activeProcess;
850 m_activeProcess = 0;
852 m_runCounter = 0;
853 m_testCounter = 0;
854 m_activeTreeItem = 0;
857 void TestRegressionWindow::testerReceivedData()
859 assert(m_activeProcess != 0);
861 QString data(m_activeProcess->readAllStandardOutput());
862 QStringList list = data.split('\n');
864 QStringList::const_iterator it = list.constBegin();
865 const QStringList::const_iterator end = list.constEnd();
867 for(; it != end; ++it)
869 QString temp = *it;
870 if(!temp.isEmpty())
871 m_processingQueue.enqueue(temp);
874 if(!m_justProcessingQueue)
876 m_justProcessingQueue = true;
877 QTimer::singleShot(50, this, SLOT(processQueue()));
881 void TestRegressionWindow::processQueue()
883 while(!m_processingQueue.isEmpty())
885 QString data = m_processingQueue.dequeue();
886 TestResult result = Unknown;
888 QString cacheName = extractTestNameFromData(data, result);
890 if(result != Unknown) // Yes, we're dealing with a test result...
892 if(cacheName.isEmpty()) // Make sure everything is alright!
894 kError() << "Couldn't extract cacheName from data=\"" << data << "\"! Ignoring!" << endl;
895 continue;
899 parseRegressionTestingOutput(data, result, cacheName);
902 m_justProcessingQueue = false;
905 void TestRegressionWindow::addToIgnores()
907 assert(m_activeTreeItem != 0);
909 QString treeItemText = pathFromItem(m_activeTreeItem);
911 // Extract directory/file name...
912 int position = treeItemText.lastIndexOf('/');
914 QString directory = treeItemText.mid(0, (position == -1 ? 0 : position));
915 QString fileName = treeItemText.mid(position + 1);
917 // Read corresponding ignore file..
918 QString ignoreFile = m_testsUrl.path() + "/tests/" + directory + "/ignore";
919 QStringList ignoreFileList = readListFile(ignoreFile);
921 if(!ignoreFileList.contains(fileName))
922 ignoreFileList.append(fileName);
924 // Commit changes...
925 writeListFile(ignoreFile, ignoreFileList);
927 // Reset icon status...
928 m_activeTreeItem->setIcon(0, m_ignorePixmap);
931 void TestRegressionWindow::removeFromIgnores()
933 assert(m_activeTreeItem != 0);
935 QString treeItemText = pathFromItem(m_activeTreeItem);
937 // Extract directory/file name...
938 int position = treeItemText.lastIndexOf('/');
940 QString directory = treeItemText.mid(0, (position == -1 ? 0 : position));
941 QString fileName = treeItemText.mid(position + 1);
943 // Read corresponding ignore file..
944 QString ignoreFile = m_testsUrl.path() + "/tests/" + directory + "/ignore";
945 QStringList ignoreFileList = readListFile(ignoreFile);
947 if(ignoreFileList.contains(fileName))
948 ignoreFileList.removeAll(fileName);
950 // Commit changes...
951 writeListFile(ignoreFile, ignoreFileList);
953 // Reset icon status...
954 m_activeTreeItem->setIcon(0, QPixmap());
957 QString TestRegressionWindow::pathFromItem(const QTreeWidgetItem *item) const
959 QString path = item->text(0);
961 QTreeWidgetItem *parent = item->parent();
962 while(parent != 0)
964 if(parent->parent() != 0)
965 path.prepend(parent->text(0) + "/"); //krazy:exclude=duoblequote_chars DOM demands chars
967 parent = parent->parent();
970 return path;
973 QString TestRegressionWindow::extractTestNameFromData(QString &data, TestResult &result) const
975 if(data.indexOf("PASS") >= 0 || data.indexOf("FAIL") >= 0)
977 // Name extraction regexps...
978 QString bracesSelector("[0-9a-zA-Z-_<>\\* +-,.:!?$'\"=/\\[\\]\\(\\)]*");
980 QRegExp expPass("PASS: (" + bracesSelector + ")"); //krazy:exclude=duoblequote_chars DOM demands chars
981 QRegExp expPassUnexpected("PASS \\(unexpected!\\): (" + bracesSelector + ")"); //krazy:exclude=duoblequote_chars DOM demands chars
983 QRegExp expFail("FAIL: (" + bracesSelector + ")"); //krazy:exclude=duoblequote_chars DOM demands chars
984 QRegExp expFailKnown("FAIL \\(known\\): (" + bracesSelector + ")"); //krazy:exclude=duoblequote_chars DOM demands chars
986 // Extract name of test... (while using regexps as rare as possible!)
987 int pos = -1;
988 QString test;
990 QRegExp cleanTest(" \\[" + bracesSelector + "\\]");
992 pos = expPass.indexIn(data);
993 if(pos > -1) { test = expPass.cap(1); result = Pass; }
995 if(result == Unknown)
997 pos = expPassUnexpected.indexIn(data);
998 if(pos > -1) { test = expPassUnexpected.cap(1); result = PassUnexpected; }
1001 if(result == Unknown)
1003 pos = expFail.indexIn(data);
1004 if(pos > -1) { test = expFail.cap(1); result = Fail; }
1007 if(result == Unknown)
1009 pos = expFailKnown.indexIn(data);
1010 if(pos > -1) { test = expFailKnown.cap(1); result = FailKnown; }
1013 if(!test.isEmpty() && result != Unknown) // Got information about test...
1015 // Clean up first, so we only get the file name...
1016 test.replace(cleanTest, QString());
1018 // Extract cached directory/filename pair...
1019 int lastSlashPos = test.lastIndexOf('/');
1021 QString cachedDirectory = (lastSlashPos > 0 ? test.mid(0, lastSlashPos) : QString());
1022 QString cachedFilename = test.mid(lastSlashPos + 1);
1024 if(cachedDirectory == ".") // Handle cases like "./empty.html"
1025 cachedDirectory.clear();
1027 assert(m_directoryMap.constFind(cachedDirectory) != m_directoryMap.constEnd());
1029 QString cacheName = cachedDirectory + "/" + cachedFilename; //krazy:exclude=duoblequote_chars DOM demands chars
1030 if(m_itemMap.constFind(cacheName) != m_itemMap.constEnd())
1032 // Highlight test...
1033 data.replace(expPass, "<b><font color='green'>PASS:\t\\1</font></b>");
1034 data.replace(expPassUnexpected, "<b><font color='green'>PASS (unexpected!):\t\\1</font></b>");
1035 data.replace(expFail, "<b><font color='red'>FAIL:\t\\1</font></b>");
1036 data.replace(expFailKnown, "<b><font color='red'>FAIL (known):\t\\1</font></b>");
1038 return cacheName;
1043 return QString();
1046 void TestRegressionWindow::parseRegressionTestingOutput(QString data, TestResult result, const QString &cacheName)
1048 if(!cacheName.isEmpty())
1050 if(m_testCounter >= 0) // Only increment in all-tests mode...
1051 m_testCounter++;
1053 m_runCounter++; // Always increment...
1055 // Update the icon...
1056 updateItemStatus(result, m_itemMap[cacheName], cacheName);
1059 // Apply some nice formatting for the statistics...
1060 if(data.indexOf("Total") >= 0 || data.indexOf("Passes") >= 0 || data.indexOf("Tests completed") >= 0 ||
1061 data.indexOf("Errors:") >= 0 || data.indexOf("Failures:") >= 0)
1063 QRegExp expTotal("Total: ([0-9]*)");
1064 QRegExp expPasses("Passes: ([0-9 a-z\\(\\)]*)");
1065 QRegExp expErrors("Errors: ([0-9 a-z]*)");
1066 QRegExp expFailures("Failures: ([0-9 a-z\\(\\)]*)");
1068 data.replace("Tests completed.", "<br><center><h2>Tests completed.</h2></center>");
1069 data.replace(expTotal, "<b><font size='4'>Total:&nbsp;\\1</font></b>");
1070 data.replace(expPasses, "<b><font size='4' color='green'>Passes:&nbsp;\\1</font></b>");
1071 data.replace(expErrors, "<b><font size='4' color='blue'>Errors:&nbsp;\\1</font></b>");
1072 data.replace(expFailures, "<b><font size='4' color='red'>Failures:&nbsp;\\1</font></b>");
1075 if(!data.contains("</body>\n</html>")) // Don't put <br> behind </html>!
1076 data.append("<br>");
1078 // Update text edit...
1079 updateLogOutput(data);
1081 // Update progressbar...
1082 if(m_testCounter > 0)
1083 m_ui.progressBar->setValue(m_testCounter);
1086 void TestRegressionWindow::treeWidgetContextMenuRequested(const QPoint &pos)
1088 if((m_testCounter == -1 && m_activeProcess) || (m_testCounter > 0 && m_activeProcess) ||
1089 m_testsUrl.isEmpty() || m_khtmlUrl.isEmpty()) // Still processing/not ready yet...
1091 return;
1094 QTreeWidgetItem *item = m_ui.treeWidget->itemAt(pos);
1095 if(item && item != m_ui.treeWidget->topLevelItem(0))
1097 m_activeTreeItem = item;
1099 // Build & show popup menu...
1100 QMenu menu(m_ui.treeWidget);
1102 menu.addAction(SmallIcon("media-playback-start"), i18n("Run test..."), this, SLOT(runSingleTest()));
1103 menu.addSeparator();
1104 menu.addAction(SmallIcon("list-add"), i18n("Add to ignores..."), this, SLOT(addToIgnores()));
1105 menu.addAction(SmallIcon("dialog-cancel"), i18n("Remove from ignores..."), this, SLOT(removeFromIgnores()));
1107 if(!menu.exec(m_ui.treeWidget->mapToGlobal(pos)))
1108 m_activeTreeItem = 0; // Needs reset...
1112 void TestRegressionWindow::updateLogOutput(const QString &data)
1114 QTextCursor cursor = m_ui.textEdit->textCursor();
1116 // Append 'data'...
1117 m_ui.textEdit->insertHtml(data);
1119 // Keep a maximum of 100 lines in the log...
1120 const int maxLogLines = 100;
1122 long logLines = countLogLines();
1123 if(logLines > maxLogLines)
1125 cursor.movePosition(QTextCursor::Start);
1126 cursor.movePosition(QTextCursor::Down, QTextCursor::KeepAnchor, logLines - maxLogLines);
1127 cursor.removeSelectedText();
1130 // Go to end...
1131 cursor.movePosition(QTextCursor::End);
1132 m_ui.textEdit->setTextCursor(cursor);
1134 // Eventually save log output...
1135 if(!m_saveLogUrl.isEmpty())
1137 QString fileName = m_saveLogUrl.path() + "/logOutput.html";
1138 QIODevice::OpenMode fileFlags = QIODevice::WriteOnly;
1140 bool fileExists = QFileInfo(fileName).exists();
1141 if(fileExists)
1142 fileFlags |= QIODevice::Append;
1144 QFile file(fileName);
1145 if(!file.open(fileFlags))
1147 kError() << " Can't open " << fileName << endl;
1148 exit(1);
1151 if(!fileExists)
1152 file.write(QString::fromLatin1("<html>\n<body>\n").toAscii());
1154 file.write((data + "\n").toAscii()); //krazy:exclude=duoblequote_chars DOM demands chars
1155 file.close();
1157 // Reset save log url, if we reached the end...
1158 if(data.contains("</body>\n</html>"))
1159 m_saveLogUrl = KUrl();
1163 unsigned long TestRegressionWindow::countLogLines() const
1165 QTextCursor cursor = m_ui.textEdit->textCursor();
1166 cursor.movePosition(QTextCursor::Start);
1168 unsigned long lines = 0;
1169 while(cursor.movePosition(QTextCursor::Down, QTextCursor::MoveAnchor))
1170 lines++;
1172 return lines;
1175 QStringList TestRegressionWindow::readListFile(const QString &fileName) const
1177 QStringList files;
1179 QFileInfo fileInfo(fileName);
1180 if(fileInfo.exists())
1182 QFile file(fileName);
1183 if(!file.open(QIODevice::ReadOnly))
1185 kError() << " Can't open " << fileName << endl;
1186 exit(1);
1189 QString line;
1191 QTextStream fileStream(&file);
1192 while(!(line = fileStream.readLine()).isNull())
1193 files.append(line);
1195 file.close();
1198 return files;
1201 void TestRegressionWindow::writeListFile(const QString &fileName, const QStringList &content) const
1203 QFile file(fileName);
1204 if(!file.open(QIODevice::WriteOnly | QIODevice::Truncate))
1206 kError() << " Can't open " << fileName << endl;
1207 exit(1);
1210 file.write(content.join("\n").toAscii());
1211 file.close();
1214 // vim:ts=4:tw=4:noet