chrome/test/perf/frame_rate/frame_rate_tests.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <map>
   6
   7 #include "base/command_line.h"
   8 #include "base/file_util.h"
   9 #include "base/path_service.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/utf_string_conversions.h"
  12 #include "base/test/test_timeouts.h"
  13 #include "base/test/trace_event_analyzer.h"
  14 #include "chrome/common/chrome_paths.h"
  15 #include "chrome/common/chrome_switches.h"
  16 #include "chrome/test/automation/automation_proxy.h"
  17 #include "chrome/test/automation/tab_proxy.h"
  18 #include "chrome/test/perf/perf_test.h"
  19 #include "chrome/test/ui/javascript_test_util.h"
  20 #include "chrome/test/ui/ui_perf_test.h"
  21 #include "net/base/net_util.h"
  22 #include "testing/perf/perf_test.h"
  23 #include "ui/gl/gl_implementation.h"
  24 #include "ui/gl/gl_switches.h"
  25
  26 #if defined(OS_WIN)
  27 #include "base/win/windows_version.h"
  28 #endif
  29
  30 namespace {
  31
  32 enum FrameRateTestFlags {
  33   kUseGpu             = 1 << 0, // Only execute test if --enable-gpu, and verify
  34                                 // that test ran on GPU. This is required for
  35                                 // tests that run on GPU.
  36   kForceGpuComposited = 1 << 1, // Force the test to use the compositor.
  37   kDisableVsync       = 1 << 2, // Do not limit framerate to vertical refresh.
  38                                 // when on GPU, nor to 60hz when not on GPU.
  39   kUseReferenceBuild  = 1 << 3, // Run test using the reference chrome build.
  40   kInternal           = 1 << 4, // Test uses internal test data.
  41   kHasRedirect        = 1 << 5, // Test page contains an HTML redirect.
  42   kIsGpuCanvasTest    = 1 << 6  // Test uses GPU accelerated canvas features.
  43 };
  44
  45 class FrameRateTest
  46   : public UIPerfTest
  47   , public ::testing::WithParamInterface<int> {
  48  public:
  49   FrameRateTest() {
  50     show_window_ = true;
  51     dom_automation_enabled_ = true;
  52   }
  53
  54   bool HasFlag(FrameRateTestFlags flag) const {
  55     return (GetParam() & flag) == flag;
  56   }
  57
  58   bool IsGpuAvailable() const {
  59     return CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu");
  60   }
  61
  62   std::string GetSuffixForTestFlags() {
  63     std::string suffix;
  64     if (HasFlag(kForceGpuComposited))
  65       suffix += "_comp";
  66     if (HasFlag(kUseGpu))
  67       suffix += "_gpu";
  68     if (HasFlag(kDisableVsync))
  69       suffix += "_novsync";
  70     if (HasFlag(kUseReferenceBuild))
  71       suffix += "_ref";
  72     return suffix;
  73   }
  74
  75   virtual base::FilePath GetDataPath(const std::string& name) {
  76     // Make sure the test data is checked out.
  77     base::FilePath test_path;
  78     PathService::Get(chrome::DIR_TEST_DATA, &test_path);
  79     test_path = test_path.Append(FILE_PATH_LITERAL("perf"));
  80     test_path = test_path.Append(FILE_PATH_LITERAL("frame_rate"));
  81     if (HasFlag(kInternal)) {
  82       test_path = test_path.Append(FILE_PATH_LITERAL("private"));
  83     } else {
  84       test_path = test_path.Append(FILE_PATH_LITERAL("content"));
  85     }
  86     test_path = test_path.AppendASCII(name);
  87     return test_path;
  88   }
  89
  90   virtual void SetUp() {
  91     if (HasFlag(kUseReferenceBuild))
  92       UseReferenceBuild();
  93
  94     // Turn on chrome.Interval to get higher-resolution timestamps on frames.
  95     launch_arguments_.AppendSwitch(switches::kEnableBenchmarking);
  96
  97     // UI tests boot up render views starting from about:blank. This causes
  98     // the renderer to start up thinking it cannot use the GPU. To work
  99     // around that, and allow the frame rate test to use the GPU, we must
 100     // pass kAllowWebUICompositing.
 101     launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);
 102
 103     // Some of the tests may launch http requests through JSON or AJAX
 104     // which causes a security error (cross domain request) when the page
 105     // is loaded from the local file system ( file:// ). The following switch
 106     // fixes that error.
 107     launch_arguments_.AppendSwitch(switches::kAllowFileAccessFromFiles);
 108
 109     if (!HasFlag(kUseGpu)) {
 110       launch_arguments_.AppendSwitch(switches::kDisableAcceleratedCompositing);
 111       launch_arguments_.AppendSwitch(switches::kDisableExperimentalWebGL);
 112       launch_arguments_.AppendSwitch(switches::kDisableAccelerated2dCanvas);
 113     }
 114
 115     if (HasFlag(kDisableVsync))
 116       launch_arguments_.AppendSwitch(switches::kDisableGpuVsync);
 117
 118     UIPerfTest::SetUp();
 119   }
 120
 121   bool DidRunOnGpu(const std::string& json_events) {
 122     using trace_analyzer::Query;
 123     using trace_analyzer::TraceAnalyzer;
 124
 125     // Check trace for GPU accleration.
 126     scoped_ptr<TraceAnalyzer> analyzer(TraceAnalyzer::Create(json_events));
 127
 128     gfx::GLImplementation gl_impl = gfx::kGLImplementationNone;
 129     const trace_analyzer::TraceEvent* gpu_event = analyzer->FindFirstOf(
 130         Query::EventNameIs("SwapBuffers") &&
 131         Query::EventHasNumberArg("GLImpl"));
 132     if (gpu_event)
 133       gl_impl = static_cast<gfx::GLImplementation>(
 134           gpu_event->GetKnownArgAsInt("GLImpl"));
 135     return (gl_impl == gfx::kGLImplementationDesktopGL ||
 136             gl_impl == gfx::kGLImplementationEGLGLES2);
 137   }
 138
 139   void RunTest(const std::string& name) {
 140 #if defined(USE_AURA)
 141     if (!HasFlag(kUseGpu)) {
 142       printf("Test skipped, Aura always runs with GPU\n");
 143       return;
 144     }
 145 #endif
 146 #if defined(OS_WIN)
 147     if (HasFlag(kUseGpu) && HasFlag(kIsGpuCanvasTest) &&
 148         base::win::OSInfo::GetInstance()->version() == base::win::VERSION_XP) {
 149       // crbug.com/128208
 150       LOG(WARNING) << "Test skipped: GPU canvas tests do not run on XP.";
 151       return;
 152     }
 153 #endif
 154
 155     if (HasFlag(kUseGpu) && !IsGpuAvailable()) {
 156       printf("Test skipped: requires gpu. Pass --enable-gpu on the command "
 157              "line if use of GPU is desired.\n");
 158       return;
 159     }
 160
 161     // Verify flag combinations.
 162     ASSERT_TRUE(HasFlag(kUseGpu) || !HasFlag(kForceGpuComposited));
 163     ASSERT_TRUE(!HasFlag(kUseGpu) || IsGpuAvailable());
 164
 165     base::FilePath test_path = GetDataPath(name);
 166     ASSERT_TRUE(base::DirectoryExists(test_path))
 167         << "Missing test directory: " << test_path.value();
 168
 169     test_path = test_path.Append(FILE_PATH_LITERAL("test.html"));
 170
 171     scoped_refptr<TabProxy> tab(GetActiveTab());
 172     ASSERT_TRUE(tab.get());
 173
 174     // TODO(jbates): remove this check when ref builds are updated.
 175     if (!HasFlag(kUseReferenceBuild))
 176       ASSERT_TRUE(automation()->BeginTracing("test_gpu"));
 177
 178     if (HasFlag(kHasRedirect)) {
 179       // If the test file is known to contain an html redirect, we must block
 180       // until the second navigation is complete and reacquire the active tab
 181       // in order to avoid a race condition.
 182       // If the following assertion is triggered due to a timeout, it is
 183       // possible that the current test does not re-direct and therefore should
 184       // not have the kHasRedirect flag turned on.
 185       ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
 186         tab->NavigateToURLBlockUntilNavigationsComplete(
 187         net::FilePathToFileURL(test_path), 2));
 188       tab = GetActiveTab();
 189       ASSERT_TRUE(tab.get());
 190     } else {
 191       ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
 192         tab->NavigateToURL(net::FilePathToFileURL(test_path)));
 193     }
 194
 195     // Block until initialization completes
 196     // If the following assertion fails intermittently, it could be due to a
 197     // race condition caused by an html redirect. If that is the case, verify
 198     // that flag kHasRedirect is enabled for the current test.
 199     ASSERT_TRUE(WaitUntilJavaScriptCondition(
 200         tab.get(),
 201         std::wstring(),
 202         L"window.domAutomationController.send(__initialized);",
 203         TestTimeouts::large_test_timeout()));
 204
 205     if (HasFlag(kForceGpuComposited)) {
 206       ASSERT_TRUE(tab->NavigateToURLAsync(
 207         GURL("javascript:__make_body_composited();")));
 208     }
 209
 210     // Start the tests.
 211     ASSERT_TRUE(tab->NavigateToURLAsync(GURL("javascript:__start_all();")));
 212
 213     // Block until the tests completes.
 214     ASSERT_TRUE(WaitUntilJavaScriptCondition(
 215         tab.get(),
 216         std::wstring(),
 217         L"window.domAutomationController.send(!__running_all);",
 218         TestTimeouts::large_test_timeout()));
 219
 220     // TODO(jbates): remove this check when ref builds are updated.
 221     if (!HasFlag(kUseReferenceBuild)) {
 222       std::string json_events;
 223       ASSERT_TRUE(automation()->EndTracing(&json_events));
 224
 225       bool did_run_on_gpu = DidRunOnGpu(json_events);
 226       bool expect_gpu = HasFlag(kUseGpu);
 227       EXPECT_EQ(expect_gpu, did_run_on_gpu);
 228     }
 229
 230     // Read out the results.
 231     std::wstring json;
 232     ASSERT_TRUE(tab->ExecuteAndExtractString(
 233         std::wstring(),
 234         L"window.domAutomationController.send("
 235         L"JSON.stringify(__calc_results_total()));",
 236         &json));
 237
 238     std::map<std::string, std::string> results;
 239     ASSERT_TRUE(JsonDictionaryToMap(base::WideToUTF8(json), &results));
 240
 241     ASSERT_TRUE(results.find("mean") != results.end());
 242     ASSERT_TRUE(results.find("sigma") != results.end());
 243     ASSERT_TRUE(results.find("gestures") != results.end());
 244     ASSERT_TRUE(results.find("means") != results.end());
 245     ASSERT_TRUE(results.find("sigmas") != results.end());
 246
 247     std::string trace_name = "interval" + GetSuffixForTestFlags();
 248     printf("GESTURES %s: %s= [%s] [%s] [%s]\n", name.c_str(),
 249                                                 trace_name.c_str(),
 250                                                 results["gestures"].c_str(),
 251                                                 results["means"].c_str(),
 252                                                 results["sigmas"].c_str());
 253
 254     std::string mean_and_error = results["mean"] + "," + results["sigma"];
 255     perf_test::PrintResultMeanAndError(name,
 256                                        std::string(),
 257                                        trace_name,
 258                                        mean_and_error,
 259                                        "milliseconds-per-frame",
 260                                        true);
 261
 262     // Navigate back to NTP so that we can quit without timing out during the
 263     // wait-for-idle stage in test framework.
 264     EXPECT_EQ(tab->GoBack(), AUTOMATION_MSG_NAVIGATION_SUCCESS);
 265   }
 266 };
 267
 268 // Must use a different class name to avoid test instantiation conflicts
 269 // with FrameRateTest. An alias is good enough. The alias names must match
 270 // the pattern FrameRate*Test* for them to get picked up by the test bots.
 271 typedef FrameRateTest FrameRateCompositingTest;
 272
 273 // Tests that trigger compositing with a -webkit-translateZ(0)
 274 #define FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(content) \
 275 TEST_P(FrameRateCompositingTest, content) { \
 276   RunTest(#content); \
 277 }
 278
 279 INSTANTIATE_TEST_CASE_P(, FrameRateCompositingTest, ::testing::Values(
 280                         0,
 281                         kUseGpu | kForceGpuComposited,
 282                         kUseReferenceBuild,
 283                         kUseReferenceBuild | kUseGpu | kForceGpuComposited));
 284
 285 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(blank);
 286 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(googleblog);
 287
 288 typedef FrameRateTest FrameRateNoVsyncCanvasInternalTest;
 289
 290 // Tests for animated 2D canvas content with and without disabling vsync
 291 #define INTERNAL_FRAME_RATE_TEST_CANVAS_WITH_AND_WITHOUT_NOVSYNC(content) \
 292 TEST_P(FrameRateNoVsyncCanvasInternalTest, content) { \
 293   RunTest(#content); \
 294 }
 295
 296 INSTANTIATE_TEST_CASE_P(, FrameRateNoVsyncCanvasInternalTest, ::testing::Values(
 297     kInternal | kHasRedirect,
 298     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu,
 299     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu | kDisableVsync,
 300     kUseReferenceBuild | kInternal | kHasRedirect,
 301     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu,
 302     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu |
 303         kDisableVsync));
 304
 305 INTERNAL_FRAME_RATE_TEST_CANVAS_WITH_AND_WITHOUT_NOVSYNC(fishbowl)
 306
 307 typedef FrameRateTest FrameRateGpuCanvasInternalTest;
 308
 309 // Tests for animated 2D canvas content to be tested only with GPU
 310 // acceleration.
 311 // tests are run with and without Vsync
 312 #define INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(content) \
 313 TEST_P(FrameRateGpuCanvasInternalTest, content) { \
 314   RunTest(#content); \
 315 }
 316
 317 INSTANTIATE_TEST_CASE_P(, FrameRateGpuCanvasInternalTest, ::testing::Values(
 318     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu,
 319     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu | kDisableVsync,
 320     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu,
 321     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu |
 322         kDisableVsync));
 323
 324 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(fireflies)
 325 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(FishIE)
 326 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(speedreading)
 327
 328 }  // namespace