chrome/browser/extensions/api/tab_capture/tab_capture_performancetest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/basictypes.h"
   6 #include "base/command_line.h"
   7 #include "base/strings/stringprintf.h"
   8 #include "base/test/trace_event_analyzer.h"
   9 #include "base/win/windows_version.h"
  10 #include "chrome/browser/extensions/extension_apitest.h"
  11 #include "chrome/browser/extensions/extension_service.h"
  12 #include "chrome/browser/extensions/tab_helper.h"
  13 #include "chrome/browser/profiles/profile.h"
  14 #include "chrome/browser/ui/exclusive_access/fullscreen_controller.h"
  15 #include "chrome/common/chrome_switches.h"
  16 #include "chrome/test/base/test_launcher_utils.h"
  17 #include "chrome/test/base/test_switches.h"
  18 #include "chrome/test/base/tracing.h"
  19 #include "content/public/common/content_switches.h"
  20 #include "extensions/common/feature_switch.h"
  21 #include "extensions/common/features/base_feature_provider.h"
  22 #include "extensions/common/features/complex_feature.h"
  23 #include "extensions/common/features/feature.h"
  24 #include "extensions/common/features/simple_feature.h"
  25 #include "extensions/common/switches.h"
  26 #include "extensions/test/extension_test_message_listener.h"
  27 #include "testing/gtest/include/gtest/gtest.h"
  28 #include "testing/perf/perf_test.h"
  29 #include "ui/compositor/compositor_switches.h"
  30 #include "ui/gl/gl_switches.h"
  31
  32 namespace {
  33
  34 const char kExtensionId[] = "ddchlicdkolnonkihahngkmmmjnjlkkf";
  35
  36 enum TestFlags {
  37   kUseGpu              = 1 << 0, // Only execute test if --enable-gpu was given
  38                                  // on the command line.  This is required for
  39                                  // tests that run on GPU.
  40   kForceGpuComposited  = 1 << 1, // Force the test to use the compositor.
  41   kDisableVsync        = 1 << 2, // Do not limit framerate to vertical refresh.
  42                                  // when on GPU, nor to 60hz when not on GPU.
  43   kTestThroughWebRTC   = 1 << 3, // Send captured frames through webrtc
  44   kSmallWindow         = 1 << 4, // 1 = 800x600, 0 = 2000x1000
  45
  46   kScaleQualityMask    = 3 << 5, // two bits select which scaling quality
  47   kScaleQualityDefault = 0 << 5, // to use on aura.
  48   kScaleQualityFast    = 1 << 5,
  49   kScaleQualityGood    = 2 << 5,
  50   kScaleQualityBest    = 3 << 5,
  51 };
  52
  53 class TabCapturePerformanceTest
  54     : public ExtensionApiTest,
  55       public testing::WithParamInterface<int> {
  56  public:
  57   TabCapturePerformanceTest() {}
  58
  59   bool HasFlag(TestFlags flag) const {
  60     return (GetParam() & flag) == flag;
  61   }
  62
  63   bool IsGpuAvailable() const {
  64     return base::CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu");
  65   }
  66
  67   std::string ScalingMethod() const {
  68     switch (GetParam() & kScaleQualityMask) {
  69       case kScaleQualityFast:
  70         return "fast";
  71       case kScaleQualityGood:
  72         return "good";
  73       case kScaleQualityBest:
  74         return "best";
  75       default:
  76         return "";
  77     }
  78   }
  79
  80   std::string GetSuffixForTestFlags() {
  81     std::string suffix;
  82     if (HasFlag(kForceGpuComposited))
  83       suffix += "_comp";
  84     if (HasFlag(kUseGpu))
  85       suffix += "_gpu";
  86     if (HasFlag(kDisableVsync))
  87       suffix += "_novsync";
  88     if (HasFlag(kTestThroughWebRTC))
  89       suffix += "_webrtc";
  90     if (!ScalingMethod().empty())
  91       suffix += "_scale" + ScalingMethod();
  92     if (HasFlag(kSmallWindow))
  93       suffix += "_small";
  94     return suffix;
  95   }
  96
  97   void SetUp() override {
  98     EnablePixelOutput();
  99     ExtensionApiTest::SetUp();
 100   }
 101
 102   void SetUpCommandLine(base::CommandLine* command_line) override {
 103     if (!ScalingMethod().empty()) {
 104       command_line->AppendSwitchASCII(switches::kTabCaptureUpscaleQuality,
 105                                       ScalingMethod());
 106       command_line->AppendSwitchASCII(switches::kTabCaptureDownscaleQuality,
 107                                       ScalingMethod());
 108     }
 109
 110     // Some of the tests may launch http requests through JSON or AJAX
 111     // which causes a security error (cross domain request) when the page
 112     // is loaded from the local file system ( file:// ). The following switch
 113     // fixes that error.
 114     command_line->AppendSwitch(switches::kAllowFileAccessFromFiles);
 115
 116     if (HasFlag(kSmallWindow)) {
 117       command_line->AppendSwitchASCII(switches::kWindowSize, "800,600");
 118     } else {
 119       command_line->AppendSwitchASCII(switches::kWindowSize, "2000,1500");
 120     }
 121
 122     if (!HasFlag(kUseGpu))
 123       command_line->AppendSwitch(switches::kDisableGpu);
 124
 125     if (HasFlag(kDisableVsync))
 126       command_line->AppendSwitch(switches::kDisableGpuVsync);
 127
 128     command_line->AppendSwitchASCII(
 129         extensions::switches::kWhitelistedExtensionID,
 130         kExtensionId);
 131     ExtensionApiTest::SetUpCommandLine(command_line);
 132   }
 133
 134   bool PrintResults(trace_analyzer::TraceAnalyzer *analyzer,
 135                     const std::string& test_name,
 136                     const std::string& event_name,
 137                     const std::string& unit) {
 138     trace_analyzer::TraceEventVector events;
 139     trace_analyzer::Query query =
 140         trace_analyzer::Query::EventNameIs(event_name) &&
 141         (trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_BEGIN) ||
 142          trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_COMPLETE) ||
 143          trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_ASYNC_BEGIN) ||
 144          trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_FLOW_BEGIN) ||
 145          trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_INSTANT));
 146     analyzer->FindEvents(query, &events);
 147     if (events.size() < 20) {
 148       LOG(ERROR) << "Not enough events of type " << event_name << " found ("
 149                  << events.size() << ").";
 150       return false;
 151     }
 152
 153     // Ignore some events for startup/setup/caching.
 154     trace_analyzer::TraceEventVector rate_events(events.begin() + 3,
 155                                                  events.end() - 3);
 156     trace_analyzer::RateStats stats;
 157     if (!GetRateStats(rate_events, &stats, NULL)) {
 158       LOG(ERROR) << "GetRateStats failed";
 159       return false;
 160     }
 161     double mean_ms = stats.mean_us / 1000.0;
 162     double std_dev_ms = stats.standard_deviation_us / 1000.0;
 163     std::string mean_and_error = base::StringPrintf("%f,%f", mean_ms,
 164                                                     std_dev_ms);
 165     perf_test::PrintResultMeanAndError(test_name,
 166                                        GetSuffixForTestFlags(),
 167                                        event_name,
 168                                        mean_and_error,
 169                                        unit,
 170                                        true);
 171     return true;
 172   }
 173
 174   void RunTest(const std::string& test_name) {
 175     if (HasFlag(kUseGpu) && !IsGpuAvailable()) {
 176       LOG(WARNING) <<
 177           "Test skipped: requires gpu. Pass --enable-gpu on the command "
 178           "line if use of GPU is desired.";
 179       return;
 180     }
 181
 182     std::string json_events;
 183     ASSERT_TRUE(tracing::BeginTracing("gpu,gpu.capture"));
 184     std::string page = "performance.html";
 185     page += HasFlag(kTestThroughWebRTC) ? "?WebRTC=1" : "?WebRTC=0";
 186     // Ideally we'd like to run a higher capture rate when vsync is disabled,
 187     // but libjingle currently doesn't allow that.
 188     // page += HasFlag(kDisableVsync) ? "&fps=300" : "&fps=30";
 189     page += "&fps=60";
 190     ASSERT_TRUE(RunExtensionSubtest("tab_capture", page)) << message_;
 191     ASSERT_TRUE(tracing::EndTracing(&json_events));
 192     scoped_ptr<trace_analyzer::TraceAnalyzer> analyzer;
 193     analyzer.reset(trace_analyzer::TraceAnalyzer::Create(json_events));
 194
 195     // The printed result will be the average time between frames in the
 196     // browser window.
 197     bool gpu_frames = PrintResults(
 198         analyzer.get(),
 199         test_name,
 200         "RenderWidget::DidCommitAndDrawCompositorFrame",
 201         "ms");
 202     EXPECT_TRUE(gpu_frames);
 203
 204     // This prints out the average time between capture events.
 205     // As the capture frame rate is capped at 30fps, this score
 206     // cannot get any better than (lower) 33.33 ms.
 207     // TODO(ericrk): Remove the "Capture" result once we are confident that
 208     // "CaptureSucceeded" is giving the coverage we want. crbug.com/489817
 209     EXPECT_TRUE(PrintResults(analyzer.get(),
 210                              test_name,
 211                              "Capture",
 212                              "ms"));
 213
 214     // Also track the CaptureSucceeded event. Capture only indicates that a
 215     // capture was requested, but this capture may later be aborted without
 216     // running. CaptureSucceeded tracks successful frame captures.
 217     EXPECT_TRUE(
 218         PrintResults(analyzer.get(), test_name, "CaptureSucceeded", "ms"));
 219   }
 220 };
 221
 222 }  // namespace
 223
 224 IN_PROC_BROWSER_TEST_P(TabCapturePerformanceTest, Performance) {
 225   RunTest("TabCapturePerformance");
 226 }
 227
 228 // Note: First argument is optional and intentionally left blank.
 229 // (it's a prefix for the generated test cases)
 230 INSTANTIATE_TEST_CASE_P(
 231     ,
 232     TabCapturePerformanceTest,
 233     testing::Values(
 234         0,
 235         kUseGpu | kForceGpuComposited,
 236         kDisableVsync,
 237         kDisableVsync | kUseGpu | kForceGpuComposited,
 238         kTestThroughWebRTC,
 239         kTestThroughWebRTC | kUseGpu | kForceGpuComposited,
 240         kTestThroughWebRTC | kDisableVsync,
 241         kTestThroughWebRTC | kDisableVsync | kUseGpu | kForceGpuComposited));
 242
 243 #if defined(USE_AURA)
 244 // TODO(hubbe):
 245 // These are temporary tests for the purpose of determining what the
 246 // appropriate scaling quality is. Once that has been determined,
 247 // these tests will be removed.
 248
 249 const int kScalingTestBase =
 250     kTestThroughWebRTC | kDisableVsync | kUseGpu | kForceGpuComposited;
 251
 252 INSTANTIATE_TEST_CASE_P(
 253     ScalingTests,
 254     TabCapturePerformanceTest,
 255     testing::Values(
 256         kScalingTestBase | kScaleQualityFast,
 257         kScalingTestBase | kScaleQualityGood,
 258         kScalingTestBase | kScaleQualityBest,
 259         kScalingTestBase | kScaleQualityFast | kSmallWindow,
 260         kScalingTestBase | kScaleQualityGood | kSmallWindow,
 261         kScalingTestBase | kScaleQualityBest | kSmallWindow));
 262
 263 #endif  // USE_AURA