3 // Populated from: http://www.medcalc.be/manual/t-distribution.php
4 // 95% confidence for N - 1 = 4
5 var tDistribution = 2.776;
7 // The number of individual test iterations to do
10 // The type of run that we're doing (options are "runs/s" or "ms")
11 var runStyle = "runs/s";
13 // A rough estimate, in seconds, of how long it'll take each test
15 var timePerTest = runStyle === "runs/s" ? 1 : 0.5;
17 // Initialize a batch of tests
18 // name = The name of the test collection
19 this.startTest = function(name, version){
21 if ( numloaded == totalTests )
22 setTimeout( init, 100 );
25 if ( !queues[testName] ) return;
27 testNames[testID] = testName;
28 testVersions[testID] = version || 0;
29 testSummary[testID] = testSummaryNum[testID] = testDone[testID] = testNum[testID] = 0;
31 queues[testID].push(function(){
37 // Anything that you want to have run in order, but not actually test
38 this.prep = function(fn){
39 if ( !queues[testName] ) return;
40 queues[testID].push(function(){
46 // End the tests and finalize the report
47 this.endTest = function(){
48 if ( !queues[testName] ) return;
49 // Save the summary output until all the test are complete
50 queues[testID].push(function(){
56 // name = The unique name of the test
57 // num = The 'length' of the test (length of string, # of tests, etc.)
58 // fn = A function holding the test to run
59 this.test = function(name, num, fn){
60 if ( !queues[testName] ) return;
61 // Save the summary output until all the test are complete
62 var curTest = testName, curID = testID;
64 if ( arguments.length === 3 ) {
65 if ( !nameDone[name] )
69 if ( nameDone[name] != 3 )
76 time += timePerTest * numTests;
80 // Don't execute the test immediately
81 queues[testID].push(function(){
83 var times = [], start, pos = 0, cur;
85 setTimeout(function(){
88 if ( doShark(name) ) {
93 start = (new Date()).getTime();
95 if ( runStyle === "runs/s" ) {
98 cur = (new Date()).getTime();
100 while ( (cur - start) < 1000 ) {
102 cur = (new Date()).getTime();
107 cur = (new Date()).getTime();
110 if ( doShark(name) ) {
115 // For making Median and Variance
116 if ( runStyle === "runs/s" ) {
117 times.push( (runs * 1000) / (cur - start) );
119 times.push( cur - start );
122 alert("FAIL " + name + " " + num + e);
126 if ( pos < numTests ) {
128 updateTestPos({curID: curID, collection: testNames[curID], version: testVersions[curID]});
131 if ( ++pos < numTests ) {
132 setTimeout( arguments.callee, 1 );
135 var data = compute( times, numTests );
138 data.collection = testNames[curID];
139 data.version = testVersions[curID];
150 function compute(times, runs){
151 var results = {runs: runs}, num = times.length;
153 times = times.sort(function(a,b){
160 for ( var i = 0; i < num; i++ )
161 results.sum += times[i];
164 results.min = times[0];
167 results.max = times[ num - 1 ];
170 results.mean = results.sum / num;
173 results.median = num % 2 == 0 ?
174 (times[Math.floor(num/2)] + times[Math.ceil(num/2)]) / 2 :
175 times[Math.round(num/2)];
178 results.variance = 0;
180 for ( var i = 0; i < num; i++ )
181 results.variance += Math.pow(times[i] - results.mean, 2);
183 results.variance /= num - 1;
185 // Make Standard Deviation
186 results.deviation = Math.sqrt( results.variance );
188 // Compute Standard Errors Mean
189 results.sem = (results.deviation / Math.sqrt(results.runs)) * tDistribution;
192 results.error = ((results.sem / results.mean) * 100) || 0;
201 // The number of test files to load
204 // The number of test files loaded
207 // Queue of functions to run
212 dromaeo: "Dromaeo JavaScript Tests",
213 sunspider: "SunSpider JavaScript Tests",
214 "v8": "V8 JavaScript Tests",
215 dom: "DOM Core Tests",
216 jslib: "JavaScript Library Tests",
217 cssquery: "CSS Selector Tests"
225 var testVersions = {};
231 var title, testName, testID, testSummary = {} , testSummaryNum = {}, maxTotal = 0, maxTotalNum = 0;
233 var automated = false;
234 var post_json = false;
236 // Query String Parsing
237 var search = window.limitSearch || (window.location.search || "?").substr(1);
239 search = search.replace(/&runStyle=([^&]+)/, function(all, type){
244 var parts = search.split("&");
246 if ( parts[0] === "recommended" ) {
247 parts[0] = "dromaeo|sunspider|v8|dom|jslib";
250 var none = !parts[0] || parts[0].match(/=/);
251 var filter = parts.length && !parts[0].match(/=/) && parts[0] !== "all" ?
252 new RegExp(parts.shift(), "i") :
255 // To enable shark debugging add &shark to the end of the URL
256 var doShark = function(name) { return false; };
257 for ( var i = 0; i < parts.length; i++ ) {
258 var m = /^shark(?:=(.*))?$/.exec(parts[i]);
260 if (m[1] === undefined) {
261 doShark = function(name) { return true; };
264 var sharkMatch = new RegExp(m[1]);
265 doShark = function(name) {
266 return sharkMatch.test(name);
271 m = /^numTests=(\d+)$/.exec(parts[i]);
273 numTests = Number(m[1]);
275 if (/^automated$/.exec(parts[i]))
277 if (/^post_json$/.exec(parts[i]))
282 var id = search.match(/id=([\d,]+)/);
285 $("#overview").hide();
289 var cat = filter.toString().slice(1,-2);
291 if ( catnames[cat] ) {
292 $("#overview span:first").html( catnames[cat] );
294 if ( catnames[cat].length > 22 ) {
295 $("#overview span:first").css("font-size", 22);
301 jQuery.getJSON("tests/MANIFEST.json", function(json){
306 for ( var name in tests )
307 // Don't load tests that we aren't looking for
308 if ( filter.test( name ) )
311 names = names.sort(function(a, b){
312 return tests[a].name < tests[b].name ? -1 :
313 tests[a].name == tests[b].name ? 0 : 1;
316 // Check if we're loading a specific result set
320 url: "store.php?id=" + id[1],
322 success: function(data){
323 resultsLoaded(id[1], data);
327 // Otherwise we're loading a normal set of tests
329 $("#wrapper").append("<br style='clear:both;'/><center><a href='?" + names.join("|") + "'>Re-run tests</a></center>");
331 for ( var i = 0; i < names.length; i++ ) (function(name){
332 var test = tests[name];
340 // Check if we're loading an HTML file
341 if ( test.file.match(/html$/) ) {
342 var iframe = document.createElement("iframe");
343 iframe.style.height = "1px";
344 iframe.style.width = "1px";
345 iframe.src = "tests/" + test.file;
346 document.body.appendChild( iframe );
348 // Otherwise we're loading a pure-JS test
350 jQuery.getScript("tests/" + test.file);
357 // Remove the next test from the queue and execute it
359 if ( interval && queue.length ) {
362 } else if ( queue.length == 0 ) {
366 $("#overview input").remove();
369 if ( window.limitSearch ) {
370 var summary = (runStyle === "runs/s" ? Math.pow(Math.E, maxTotal / maxTotalNum) : maxTotal).toFixed(2);
372 if ( typeof tpRecordTime !== "undefined" ) {
373 tpRecordTime( summary );
376 var pre = document.createElement("pre");
377 pre.style.display = "none";
378 pre.innerHTML = "__start_report" + summary + "__end_report";
379 document.body.appendChild( pre );
382 if ( typeof goQuitApplication !== "undefined" ) {
386 } else if ( dataStore && dataStore.length ) {
388 $("body").addClass("alldone");
389 var div = jQuery("<div class='results'>Saving...</div>").insertBefore("#overview");
393 data: "data=" + encodeURIComponent(JSON.stringify(dataStore)) + "&style=" + runStyle,
394 success: function(id){
395 var url = window.location.href.replace(/\?.*$/, "") + "?id=" + id;
396 div.html("Results saved. You can access them at a later time at the following URL:<br/><strong><a href='" + url + "'>" + url + "</a></strong></div>");
399 } else if (post_json) {
403 data: "data=" + encodeURIComponent(JSON.stringify(window.automation.GetResults()))
407 window.automation.SetDone();
413 function updateTimebar(){
414 $("#timebar").html("<span><strong>" + (runStyle === "runs/s" ? Math.pow(Math.E, maxTotal / maxTotalNum) : maxTotal).toFixed(2) + "</strong>" + runStyle + " (Total)</span>");
417 // Run once all the test files are fully loaded
419 for ( var n = 0; n < names.length; n++ ) {
420 queue = queue.concat( queues[ names[n] ] );
439 this.value = "Pause";
445 .click(function(){});
450 if ( window.limitSearch ) {
455 function initTest(curID){
456 $("<div class='result-item'></div>")
457 .append( testElems[ curID ] )
458 .append( "<p>" + (tests[curID] ? tests[ curID ].desc : "") + "<br/><a href='" +
459 (tests[curID] && tests[curID].origin ? tests[ curID ].origin[1] : "") + "'>Origin</a>, <a href='tests/" +
460 (tests[curID] ? tests[ curID ].file : "") + "'>Source</a>, <b>Tests:</b> " +
461 (tests[curID] && tests[curID].tags ? tests[ curID ].tags.join(", ") : "") + "</p>" )
462 .append( "<ol class='results'></ol>" )
466 function resultsLoaded(id, datas){
471 var overview = document.getElementById("overview");
473 for ( var d = 0; d < datas.length; d++ ) {
476 runStyle = data.style;
478 if ( datas.length == 1 ) {
479 $("#overview").before("<div class='results'>Viewing test run #" + id +
480 ", run on: " + data.created_at + " by:<br>" + data.useragent + "</div>");
483 runs[data.id] = data;
484 runs[data.id].mean = 0;
485 runs[data.id].error = 0;
486 runs[data.id].num = 0;
487 runs[data.id].name = (data.useragent.match(/(MSIE [\d.]+)/) ||
488 data.useragent.match(/((?:WebKit|Firefox|Shiretoko|Opera)\/[\w.]+)/) || [0,data.id])[1];
490 for ( var i = 0; i < data.results.length; i++ ) {
491 var result = data.results[i];
492 var curID = result.collection;
493 var run = result.run_id;
495 result.version += data.style;
497 if ( !results[curID] )
498 results[curID] = {tests:{}, total:{}, version: result.version};
500 if ( results[curID].version == result.version ) {
501 if ( !results[curID].total[run] ) {
502 results[curID].total[run] = {max:0, mean:0, median:0, min:0, deviation:0, error:0, num:0};
503 results[curID].tests[run] = [];
506 result.error = ((((result.deviation / Math.sqrt(result.runs)) * tDistribution) / result.mean) * 100) || 0;
507 results[curID].tests[run].push( result );
509 var error = (parseFloat(result.error) / 100) * parseFloat(result.mean);
510 error = (runStyle === "ms" ? error : error == 0 ? 0 : Math.log(error));
511 var total = results[curID].total[run];
514 for ( var type in total ) {
515 if ( type == "error" ) {
516 total.error += error;
517 } else if ( type == "mean" ) {
518 total.mean += (runStyle === "ms" ? parseFloat(result.mean) : Math.log(parseFloat(result.mean)));
519 } else if ( type !== "num" ) {
520 total[type] += parseFloat(result[type]);
525 runs[run].mean += runStyle === "ms" ? parseFloat(result.mean) : Math.log(parseFloat(result.mean));
526 runs[run].error += error;
533 if ( datas.length == 1 ) {
534 $("body").addClass("alldone");
536 for ( var i = 0; i < data.results.length; i++ ) {
537 var item = data.results[i];
538 var result = item.curID = item.collection;
540 if ( !filter.test(result) )
543 if ( !testElems[result] ) {
544 runTests.push(result);
549 // Compute Standard Errors Mean
550 item.sem = (item.deviation / Math.sqrt(item.runs)) * tDistribution;
553 item.error = ((item.sem / item.mean) * 100) || 0;
557 // testDone, testNum, testSummary
558 testDone[ result ] = numTests - 1;
559 testNum[ result ] = 1;
561 updateTestPos( item );
564 $("div.result-item").addClass("done");
566 totalTime = time = timePerTest;
569 $("#overview input").remove();
572 // Remove results where there is only one comparison set
573 for ( var id in results ) {
576 for ( var ntest in results[id].tests ) {
588 var preoutput = "<tr><td></td>";
589 for ( var run in runs )
590 preoutput += "<th><a href='?id=" + run + "'>" + runs[run].name + "</a></th>";
591 //preoutput += "<th>Winning %</th></tr>";
592 preoutput += "</tr>";
594 for ( var result in results ) {
595 // Skip results that we're filtering out
596 if ( !filter.test(result) )
599 runTests.push(result);
601 if ( runStyle === "runs/s" ) {
602 for ( var run in runs ) {
603 var mean = results[result].total[run].mean - 0;
604 var error = results[result].total[run].error - 0;
606 mean = Math.pow(Math.E, mean / results[result].total[run].num);
607 error = Math.pow(Math.E, error / results[result].total[run].num);
608 results[result].total[run].mean = mean;
609 results[result].total[run].error = error;
613 var name = tests[result] ? tests[result].name : result;
614 var tmp = processWinner(results[result].total);
616 output += "<tr><th class='name'><span onclick='toggleResults(this.nextSibling);'>▶ </span>" +
617 "<a href='' onclick='return toggleResults(this);'>" + name + "</a></th>";
619 for ( var run in runs ) {
620 var mean = results[result].total[run].mean - 0;
621 var error = results[result].total[run].error - 0;
623 output += "<td class='" + (tmp[run] || '') + "'>" + mean.toFixed(2) + "<small>" + runStyle + " ±" + ((error / mean) * 100).toFixed(2) + "%</small></td>";
629 var _tests = results[result].tests, _data = _tests[run], _num = _data.length;
630 for ( var i = 0; i < _num; i++ ) {
631 output += "<tr class='onetest hidden'><td><small>" + _data[i].name + "</small></td>";
632 for ( var run in runs ) {
633 output += "<td>" + (_tests[run][i].mean - 0).toFixed(2) + "<small>" + runStyle + " ±" + (_tests[run][i].error - 0).toFixed(2) + "%</small></td>";
635 output += "<td></td></tr>";
639 if ( runStyle === "runs/s" ) {
640 for ( var run in runs ) {
641 runs[run].mean = Math.pow(Math.E, runs[run].mean / runs[run].num);
642 runs[run].error = Math.pow(Math.E, runs[run].error / runs[run].num);
646 var tmp = processWinner(runs);
647 var totaloutput = "";
649 if ( runStyle === "ms" ) {
650 totaloutput += "<tr><th class='name'>Total:</th>";
652 totaloutput += "<tr><th class='name'>Total Score:</th>";
655 for ( var run in runs ) {
656 totaloutput += "<th class='name " + (tmp[run] || '') + "' title='" + (tmp[run + "title"] || '') + "'>" + runs[run].mean.toFixed(2) + "<small>" + runStyle + " ±" + ((runs[run].error / runs[run].mean) * 100).toFixed(2) + "%</small></th>";
660 totaloutput += "</tr>";
662 overview.className = "";
663 overview.innerHTML = "<div class='resultwrap'><table class='results'>" + preoutput + totaloutput + output + totaloutput + "</table>" + (excluded.length ? "<div style='text-align:left;'><small><b>Excluded Tests:</b> " + excluded.sort().join(", ") + "</small></div>" : "") + "</div>";
666 $("#wrapper").append("<center><a href='?" + runTests.join("|") + "'>Re-run tests</a></center>");
668 function showWinner(tmp){
669 if ( datas.length > 1 ) {
671 output += "<th>Tie</th>";
673 output += "<th>" + tmp.diff + "%</th>";
678 this.toggleResults = function(elem){
679 var span = elem.previousSibling;
682 elem = elem.parentNode.parentNode.nextSibling;
684 span.innerHTML = elem.className.indexOf("hidden") < 0 ? "▶ " : "▼ ";
686 while ( elem && elem.className.indexOf("onetest") >= 0 ) {
687 elem.className = "onetest" + (elem.className.indexOf("hidden") >= 0 ? " " : " hidden");
688 elem = elem.nextSibling;
694 function updateTime(){
696 $("#left").html(Math.floor(time / 60) + ":" + (time % 60 < 10 ? "0" : "" ) + Math.floor(time % 60));
698 var w = ((totalTime - time) / totalTime) * 100;
700 $("#timebar").width((w < 1 ? 1 : w) + "%");
703 function logTest(data){
704 // Keep a running summary going
705 data.mean = parseFloat(data.mean);
706 var mean = (runStyle === "runs/s" ? Math.log(data.mean) : data.mean);
707 testSummary[data.curID] = (testSummary[data.curID] || 0) + mean;
708 testSummaryNum[data.curID] = (testSummaryNum[data.curID] || 0) + 1;
713 testDone[data.curID]--;
716 testElems[data.curID].next().next().append("<li><b>" + data.name +
717 ":</b> " + data.mean.toFixed(2) + "<small>" + runStyle + " ±" + data.error.toFixed(2) + "%</small></li>");
719 dataStore.push(data);
722 function updateTestPos(data, update){
724 testDone[data.curID]++;
726 var per = (testDone[data.curID] / (testNum[data.curID] * numTests)) * 100;
731 var mean = (runStyle === "runs/s" ?
732 Math.pow(Math.E, testSummary[data.curID] / testSummaryNum[data.curID]) :
733 testSummary[data.curID]);
735 testElems[data.curID].html("<b>" + (tests[data.curID] ? tests[data.curID].name : data.curID) +
736 ":</b> <div class='bar'><div style='width:" +
737 per + "%;'>" + (per >= 100 ? "<span>" + mean.toFixed(2) + runStyle + "</span>" : "") + "</div></div>");
739 if ( per >= 100 && testSummary[data.curID] > 0 ) {
740 testElems[data.curID].parent().addClass("done");
744 function processWinner(data){
745 var minVal = -1, min2Val = -1, min, min2;
747 for ( var i in data ) {
748 var total = data[i].mean;
749 if ( minVal == -1 || (runStyle === "ms" && total <= minVal || runStyle === "runs/s" && total >= minVal) ) {
754 } else if ( min2Val == -1 || (runStyle === "ms" && total <= minVal || runStyle === "runs/s" && total >= min2Val) ) {
760 var tieVal = (runStyle === "ms" ? minVal : min2Val) + data[min].error + data[min2].error;
764 diff: runStyle === "ms" ?
765 -1 * Math.round((1 - (min2Val / minVal)) * 100) :
766 Math.round(((minVal / min2Val) - 1) * 100),
767 tie: minVal == min2Val || (runStyle === "ms" ? tieVal >= min2Val : tieVal >= minVal)
770 ret.tie = ret.tie || ret.diff == 0;
775 ret[ min + 'title' ] = "Tied with another run.";
776 ret[ min2 + 'title' ] = "Tied with another run.";
778 ret[ min ] = 'winner';
779 if ( min2Val > -1 ) {
780 ret[ min + 'title' ] = "Won by " + ret.diff + "%.";
787 function makeElem(testID){
789 if ( tests[testID] ) {
790 var cat = tests[testID].category, catsm = cat.replace(/[^\w]/g, "-");
791 if ( !$("#" + catsm).length ) {
792 $("#main").append("<h2 id='" + catsm + "' class='test'><a href='?cat=" + cat +"'>" + cat + '</a><div class="bar"><div id="timebar" style="width:25%;"><span class="left">Est. Time: <strong id="left">0:00</strong></span></div></div>');
797 testElems[testID] = $("<div class='test'></div>")
799 var next = jQuery(this).next().next();
800 if ( next.children().length == 0 ) return;
801 var display = next.css("display");
802 next.css("display", display == 'none' ? 'block' : 'none');
805 updateTestPos({curID: testID, collection: tests[testID] ? tests[testID].name : testID, version: testVersions[testID]}, true);
809 // Add some more stuff if running in automated mode.
810 window.automation = {}
811 window.automation.SetDone = function() {
812 console.log("Total: " + this.GetScore());
813 window.document.cookie = "__done=1; path=/";
815 window.automation.GetScore = function() {
816 return (runStyle === "runs/s" ? Math.pow(Math.E, maxTotal / maxTotalNum) : maxTotal).toString();
818 window.automation.GetResults = function() {
821 function normalizeName(name) {
822 // At least for ui_tests, dots are not allowed.
823 return name.replace(".", "_");
825 function appendToAggregated(name, value) {
826 name = normalizeName(name);
827 (aggregated[name] || (aggregated[name] = [])).push(Math.log(value));
830 for (var i = 0; i < dataStore.length; i++) {
831 var data = dataStore[i];
832 var topName = data.collection.split("-", 1)[0];
833 appendToAggregated(topName, data.mean);
834 appendToAggregated(data.collection, data.mean);
835 results[normalizeName(data.collection + "/" + data.name)] = data.mean.toString();
838 for (var name in aggregated) {
839 var means = aggregated[name];
841 for (var i = 0; i < means.length; i++) sum += means[i];
842 results[name] = Math.pow(Math.E, sum/means.length).toString();