chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/base64.h"
   6 #include "base/command_line.h"
   7 #include "base/environment.h"
   8 #include "base/files/file.h"
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/path_service.h"
  12 #include "base/process/launch.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_split.h"
  15 #include "base/strings/stringprintf.h"
  16 #include "base/test/test_timeouts.h"
  17 #include "base/time/time.h"
  18 #include "chrome/browser/chrome_notification_types.h"
  19 #include "chrome/browser/infobars/infobar_service.h"
  20 #include "chrome/browser/media/media_stream_infobar_delegate.h"
  21 #include "chrome/browser/media/webrtc_browsertest_base.h"
  22 #include "chrome/browser/media/webrtc_browsertest_common.h"
  23 #include "chrome/browser/profiles/profile.h"
  24 #include "chrome/browser/ui/browser.h"
  25 #include "chrome/browser/ui/browser_tabstrip.h"
  26 #include "chrome/browser/ui/tabs/tab_strip_model.h"
  27 #include "chrome/common/chrome_switches.h"
  28 #include "chrome/test/base/in_process_browser_test.h"
  29 #include "components/infobars/core/infobar.h"
  30 #include "content/public/browser/notification_service.h"
  31 #include "content/public/test/browser_test_utils.h"
  32 #include "media/base/media_switches.h"
  33 #include "net/test/embedded_test_server/embedded_test_server.h"
  34 #include "net/test/python_utils.h"
  35 #include "testing/perf/perf_test.h"
  36 #include "ui/gl/gl_switches.h"
  37
  38 static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
  39 #if defined(OS_WIN)
  40     FILE_PATH_LITERAL("frame_analyzer.exe");
  41 #else
  42     FILE_PATH_LITERAL("frame_analyzer");
  43 #endif
  44
  45 static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
  46 #if defined(OS_WIN)
  47     FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
  48 #else
  49     FILE_PATH_LITERAL("rgba_to_i420_converter");
  50 #endif
  51
  52 static const base::FilePath::CharType kCapturedYuvFileName[] =
  53     FILE_PATH_LITERAL("captured_video.yuv");
  54 static const base::FilePath::CharType kStatsFileName[] =
  55     FILE_PATH_LITERAL("stats.txt");
  56 static const char kMainWebrtcTestHtmlPage[] =
  57     "/webrtc/webrtc_jsep01_test.html";
  58 static const char kCapturingWebrtcHtmlPage[] =
  59     "/webrtc/webrtc_video_quality_test.html";
  60
  61 static const struct VideoQualityTestConfig {
  62   const char* test_name;
  63   int width;
  64   int height;
  65   const base::FilePath::CharType* reference_video;
  66   const char* constraints;
  67 } kVideoConfigurations[] = {
  68   { "360p", 640, 360,
  69     test::kReferenceFileName360p,
  70     WebRtcTestBase::kAudioVideoCallConstraints360p },
  71     // TODO(phoglund): Temporarily disabled on Windows because 720p has become
  72     // very, very slow on Windows (550s).
  73     // See https://code.google.com/p/webrtc/issues/detail?id=4986.
  74 #if !defined(OS_WIN)
  75     { "720p", 1280, 720,
  76     test::kReferenceFileName720p,
  77     WebRtcTestBase::kAudioVideoCallConstraints720p },
  78 #endif
  79 };
  80
  81 // Test the video quality of the WebRTC output.
  82 //
  83 // Prerequisites: This test case must run on a machine with a chrome playing
  84 // the video from the reference files located in GetReferenceFilesDir().
  85 // The file kReferenceY4mFileName.kY4mFileExtension is played using a
  86 // FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
  87 // comparison.
  88 //
  89 // You must also compile the chromium_builder_webrtc target before you run this
  90 // test to get all the tools built.
  91 //
  92 // The external compare_videos.py script also depends on two external
  93 // executables which must be located in the PATH when running this test.
  94 // * zxing (see the CPP version at https://code.google.com/p/zxing)
  95 // * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
  96 //
  97 // The test runs several custom binaries - rgba_to_i420 converter and
  98 // frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
  99 // test also runs a stand alone Python implementation of a WebSocket server
 100 // (pywebsocket) and a barcode_decoder script.
 101 class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
 102     public testing::WithParamInterface<VideoQualityTestConfig> {
 103  public:
 104   WebRtcVideoQualityBrowserTest()
 105       : environment_(base::Environment::Create()) {
 106     test_config_ = GetParam();
 107   }
 108
 109   void SetUpInProcessBrowserTestFixture() override {
 110     DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
 111
 112     ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir());
 113   }
 114
 115   void SetUpCommandLine(base::CommandLine* command_line) override {
 116     // Set up the command line option with the expected file name. We will check
 117     // its existence in HasAllRequiredResources().
 118     webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
 119         .Append(test_config_.reference_video)
 120         .AddExtension(test::kY4mFileExtension);
 121     command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
 122                                    webrtc_reference_video_y4m_);
 123     command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
 124
 125     // The video playback will not work without a GPU, so force its use here.
 126     command_line->AppendSwitch(switches::kUseGpuInTests);
 127   }
 128
 129   // Writes all frames we've captured so far by grabbing them from the
 130   // javascript and writing them to the temporary work directory.
 131   void WriteCapturedFramesToWorkingDir(content::WebContents* capturing_tab) {
 132     int num_frames = 0;
 133     std::string response =
 134         ExecuteJavascript("getTotalNumberCapturedFrames()", capturing_tab);
 135     ASSERT_TRUE(base::StringToInt(response, &num_frames)) <<
 136         "Failed to retrieve frame count: got " << response;
 137     ASSERT_NE(0, num_frames) << "Failed to capture any frames.";
 138
 139     for (int i = 0; i < num_frames; i++) {
 140       std::string base64_encoded_frame =
 141           ExecuteJavascript(base::StringPrintf("getOneCapturedFrame(%d)", i),
 142                             capturing_tab);
 143       std::string decoded_frame;
 144       ASSERT_TRUE(base::Base64Decode(base64_encoded_frame, &decoded_frame))
 145           << "Failed to decode frame data '" << base64_encoded_frame << "'.";
 146
 147       std::string file_name = base::StringPrintf("frame_%04d", i);
 148       base::File frame_file(GetWorkingDir().AppendASCII(file_name),
 149                             base::File::FLAG_CREATE | base::File::FLAG_WRITE);
 150       size_t written = frame_file.Write(0, decoded_frame.c_str(),
 151                                         decoded_frame.length());
 152       ASSERT_EQ(decoded_frame.length(), written);
 153     }
 154   }
 155
 156   // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
 157   // which should contain frames of size |width| x |height|.
 158   //
 159   // The rgba_to_i420_converter is part of the webrtc_test_tools target which
 160   // should be build prior to running this test. The resulting binary should
 161   // live next to Chrome.
 162   bool RunARGBtoI420Converter(int width,
 163                               int height,
 164                               const base::FilePath& captured_video_filename) {
 165     base::FilePath path_to_converter =
 166         GetBrowserDir().Append(kArgbToI420ConverterExecutable);
 167
 168     if (!base::PathExists(path_to_converter)) {
 169       LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
 170           << path_to_converter.value()
 171           << ". Try building the chromium_builder_webrtc target.";
 172       return false;
 173     }
 174
 175     base::CommandLine converter_command(path_to_converter);
 176     converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
 177     converter_command.AppendSwitchPath("--output_file",
 178                                        captured_video_filename);
 179     converter_command.AppendSwitchASCII("--width", base::IntToString(width));
 180     converter_command.AppendSwitchASCII("--height", base::IntToString(height));
 181     converter_command.AppendSwitchASCII("--delete_frames", "true");
 182
 183     // We produce an output file that will later be used as an input to the
 184     // barcode decoder and frame analyzer tools.
 185     DVLOG(0) << "Running " << converter_command.GetCommandLineString();
 186     std::string result;
 187     bool ok = base::GetAppOutput(converter_command, &result);
 188     DVLOG(0) << "Output was:\n\n" << result;
 189     return ok;
 190   }
 191
 192   // Compares the |captured_video_filename| with the |reference_video_filename|.
 193   //
 194   // The barcode decoder decodes the captured video containing barcodes overlaid
 195   // into every frame of the video (produced by rgba_to_i420_converter). It
 196   // produces a set of PNG images and a |stats_file| that maps each captured
 197   // frame to a frame in the reference video. The frames should be of size
 198   // |width| x |height|.
 199   // All measurements calculated are printed as perf parsable numbers to stdout.
 200   bool CompareVideosAndPrintResult(
 201       const char* test_label,
 202       int width,
 203       int height,
 204       const base::FilePath& captured_video_filename,
 205       const base::FilePath& reference_video_filename,
 206       const base::FilePath& stats_file) {
 207
 208     base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
 209         GetBrowserDir().Append(kFrameAnalyzerExecutable));
 210     base::FilePath path_to_compare_script = GetSourceDir().Append(
 211         FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
 212
 213     if (!base::PathExists(path_to_analyzer)) {
 214       LOG(ERROR) << "Missing frame analyzer: should be in "
 215           << path_to_analyzer.value()
 216           << ". Try building the chromium_builder_webrtc target.";
 217       return false;
 218     }
 219     if (!base::PathExists(path_to_compare_script)) {
 220       LOG(ERROR) << "Missing video compare script: should be in "
 221           << path_to_compare_script.value();
 222       return false;
 223     }
 224
 225     base::FilePath path_to_zxing = test::GetToolForPlatform("zxing");
 226     if (!base::PathExists(path_to_zxing)) {
 227       LOG(ERROR) << "Missing zxing: should be in " << path_to_zxing.value();
 228       return false;
 229     }
 230     base::FilePath path_to_ffmpeg = test::GetToolForPlatform("ffmpeg");
 231     if (!base::PathExists(path_to_ffmpeg)) {
 232       LOG(ERROR) << "Missing ffmpeg: should be in " << path_to_ffmpeg.value();
 233       return false;
 234     }
 235
 236     // Note: don't append switches to this command since it will mess up the
 237     // -u in the python invocation!
 238     base::CommandLine compare_command(base::CommandLine::NO_PROGRAM);
 239     EXPECT_TRUE(GetPythonCommand(&compare_command));
 240
 241     compare_command.AppendArgPath(path_to_compare_script);
 242     compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
 243     compare_command.AppendArg("--ref_video");
 244     compare_command.AppendArgPath(reference_video_filename);
 245     compare_command.AppendArg("--test_video");
 246     compare_command.AppendArgPath(captured_video_filename);
 247     compare_command.AppendArg("--frame_analyzer");
 248     compare_command.AppendArgPath(path_to_analyzer);
 249     compare_command.AppendArg("--yuv_frame_width");
 250     compare_command.AppendArg(base::IntToString(width));
 251     compare_command.AppendArg("--yuv_frame_height");
 252     compare_command.AppendArg(base::IntToString(height));
 253     compare_command.AppendArg("--zxing_path");
 254     compare_command.AppendArgPath(path_to_zxing);
 255     compare_command.AppendArg("--ffmpeg_path");
 256     compare_command.AppendArgPath(path_to_ffmpeg);
 257     compare_command.AppendArg("--stats_file");
 258     compare_command.AppendArgPath(stats_file);
 259
 260     DVLOG(0) << "Running " << compare_command.GetCommandLineString();
 261     std::string output;
 262     bool ok = base::GetAppOutput(compare_command, &output);
 263
 264     // Print to stdout to ensure the perf numbers are parsed properly by the
 265     // buildbot step. The tool should print a handful RESULT lines.
 266     printf("Output was:\n\n%s\n", output.c_str());
 267     bool has_result_lines = output.find("RESULT") != std::string::npos;
 268     if (!ok || !has_result_lines) {
 269       LOG(ERROR) << "Failed to compare videos; see output above to see what "
 270                  << "the error was.";
 271       return false;
 272     }
 273     return true;
 274   }
 275
 276  protected:
 277   VideoQualityTestConfig test_config_;
 278
 279   base::FilePath GetWorkingDir() {
 280     return temp_working_dir_.path();
 281   }
 282
 283  private:
 284   base::FilePath GetSourceDir() {
 285     base::FilePath source_dir;
 286     PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
 287     return source_dir;
 288   }
 289
 290   base::FilePath GetBrowserDir() {
 291     base::FilePath browser_dir;
 292     EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
 293     return browser_dir;
 294   }
 295
 296   scoped_ptr<base::Environment> environment_;
 297   base::FilePath webrtc_reference_video_y4m_;
 298   base::ScopedTempDir temp_working_dir_;
 299 };
 300
 301 INSTANTIATE_TEST_CASE_P(
 302     WebRtcVideoQualityBrowserTests,
 303     WebRtcVideoQualityBrowserTest,
 304     testing::ValuesIn(kVideoConfigurations));
 305
 306 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
 307                        MANUAL_TestVideoQuality) {
 308   if (OnWinXp())
 309     return;  // Fails on XP. http://crbug.com/353078.
 310
 311   ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
 312       "This is a long-running test; you must specify "
 313       "--ui-test-action-max-timeout to have a value of at least 150000.";
 314   ASSERT_TRUE(test::HasReferenceFilesInCheckout());
 315   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 316
 317   content::WebContents* left_tab =
 318       OpenPageAndGetUserMediaInNewTabWithConstraints(
 319           embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
 320           test_config_.constraints);
 321   content::WebContents* right_tab =
 322       OpenPageAndGetUserMediaInNewTabWithConstraints(
 323           embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
 324           test_config_.constraints);
 325
 326   SetupPeerconnectionWithLocalStream(left_tab);
 327   SetupPeerconnectionWithLocalStream(right_tab);
 328
 329   NegotiateCall(left_tab, right_tab);
 330
 331   // Poll slower here to avoid flooding the log with messages: capturing and
 332   // sending frames take quite a bit of time.
 333   int polling_interval_msec = 1000;
 334
 335   EXPECT_TRUE(test::PollingWaitUntil(
 336       "doneFrameCapturing()", "done-capturing", right_tab,
 337       polling_interval_msec));
 338
 339   HangUp(left_tab);
 340
 341   WriteCapturedFramesToWorkingDir(right_tab);
 342
 343   // Shut everything down to avoid having the javascript race with the analysis
 344   // tools. For instance, dont have console log printouts interleave with the
 345   // RESULT lines from the analysis tools (crbug.com/323200).
 346   chrome::CloseWebContents(browser(), left_tab, false);
 347   chrome::CloseWebContents(browser(), right_tab, false);
 348
 349   ASSERT_TRUE(RunARGBtoI420Converter(
 350       test_config_.width, test_config_.height,
 351       GetWorkingDir().Append(kCapturedYuvFileName)));
 352   ASSERT_TRUE(CompareVideosAndPrintResult(
 353       test_config_.test_name,
 354       test_config_.width,
 355       test_config_.height,
 356       GetWorkingDir().Append(kCapturedYuvFileName),
 357       test::GetReferenceFilesDir()
 358           .Append(test_config_.reference_video)
 359           .AddExtension(test::kYuvFileExtension),
 360       GetWorkingDir().Append(kStatsFileName)));
 361 }