chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <ctime>
   6
   7 #include "base/command_line.h"
   8 #include "base/files/file_util.h"
   9 #include "base/path_service.h"
  10 #include "base/process/launch.h"
  11 #include "base/scoped_native_library.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "chrome/browser/media/webrtc_browsertest_base.h"
  14 #include "chrome/browser/media/webrtc_browsertest_common.h"
  15 #include "chrome/browser/profiles/profile.h"
  16 #include "chrome/browser/ui/browser.h"
  17 #include "chrome/browser/ui/browser_tabstrip.h"
  18 #include "chrome/browser/ui/tabs/tab_strip_model.h"
  19 #include "chrome/common/chrome_paths.h"
  20 #include "chrome/common/chrome_switches.h"
  21 #include "chrome/test/base/ui_test_utils.h"
  22 #include "content/public/test/browser_test_utils.h"
  23 #include "media/base/media_switches.h"
  24 #include "net/test/embedded_test_server/embedded_test_server.h"
  25 #include "testing/perf/perf_test.h"
  26
  27 // These are relative to the reference file dir defined by
  28 // webrtc_browsertest_common.h (i.e. chrome/test/data/webrtc/resources).
  29 static const base::FilePath::CharType kReferenceFile[] =
  30 #if defined (OS_WIN)
  31     FILE_PATH_LITERAL("human-voice-win.wav");
  32 #elif defined (OS_MACOSX)
  33     FILE_PATH_LITERAL("human-voice-mac.wav");
  34 #else
  35     FILE_PATH_LITERAL("human-voice-linux.wav");
  36 #endif
  37
  38 // The javascript will load the reference file relative to its location,
  39 // which is in /webrtc on the web server. The files we are looking for are in
  40 // webrtc/resources in the chrome/test/data folder.
  41 static const char kReferenceFileRelativeUrl[] =
  42 #if defined (OS_WIN)
  43     "resources/human-voice-win.wav";
  44 #elif defined (OS_MACOSX)
  45     "resources/human-voice-mac.wav";
  46 #else
  47     "resources/human-voice-linux.wav";
  48 #endif
  49
  50 static const char kMainWebrtcTestHtmlPage[] =
  51     "/webrtc/webrtc_audio_quality_test.html";
  52
  53 // Test we can set up a WebRTC call and play audio through it.
  54 //
  55 // If you're not a googler and want to run this test, you need to provide a
  56 // pesq binary for your platform (and sox.exe on windows). Read more on how
  57 // resources are managed in chrome/test/data/webrtc/resources/README.
  58 //
  59 // This test will only work on machines that have been configured to record
  60 // their own input.
  61 //
  62 // On Linux:
  63 // 1. # sudo apt-get install pavucontrol sox
  64 // 2. For the user who will run the test: # pavucontrol
  65 // 3. In a separate terminal, # arecord dummy
  66 // 4. In pavucontrol, go to the recording tab.
  67 // 5. For the ALSA plug-in [aplay]: ALSA Capture from, change from <x> to
  68 //    <Monitor of x>, where x is whatever your primary sound device is called.
  69 // 6. Try launching chrome as the target user on the target machine, try
  70 //    playing, say, a YouTube video, and record with # arecord -f dat tmp.dat.
  71 //    Verify the recording with aplay (should have recorded what you played
  72 //    from chrome).
  73 //
  74 // Note: the volume for ALL your input devices will be forced to 100% by
  75 //       running this test on Linux.
  76 //
  77 // On Mac:
  78 // 1. Get SoundFlower: http://rogueamoeba.com/freebies/soundflower/download.php
  79 // 2. Install it + reboot.
  80 // 3. Install MacPorts (http://www.macports.org/).
  81 // 4. Install sox: sudo port install sox.
  82 // 5. (For Chrome bots) Ensure sox and rec are reachable from the env the test
  83 //    executes in (sox and rec tends to install in /opt/, which generally isn't
  84 //    in the Chrome bots' env). For instance, run
  85 //    sudo ln -s /opt/local/bin/rec /usr/local/bin/rec
  86 //    sudo ln -s /opt/local/bin/sox /usr/local/bin/sox
  87 // 6. In Sound Preferences, set both input and output to Soundflower (2ch).
  88 //    Note: You will no longer hear audio on this machine, and it will no
  89 //    longer use any built-in mics.
  90 // 7. Try launching chrome as the target user on the target machine, try
  91 //    playing, say, a YouTube video, and record with 'rec test.wav trim 0 5'.
  92 //    Stop the video in chrome and try playing back the file; you should hear
  93 //    a recording of the video (note; if you play back on the target machine
  94 //    you must revert the changes in step 3 first).
  95 //
  96 // On Windows 7:
  97 // 1. Control panel > Sound > Manage audio devices.
  98 // 2. In the recording tab, right-click in an empty space in the pane with the
  99 //    devices. Tick 'show disabled devices'.
 100 // 3. You should see a 'stero mix' device - this is what your speakers output.
 101 //    Right click > Properties.
 102 // 4. In the Listen tab for the mix device, check the 'listen to this device'
 103 //    checkbox. Ensure the mix device is the default recording device.
 104 // 5. Launch chrome and try playing a video with sound. You should see
 105 //    in the volume meter for the mix device. Configure the mix device to have
 106 //    50 / 100 in level. Also go into the playback tab, right-click Speakers,
 107 //    and set that level to 50 / 100. Otherwise you will get distortion in
 108 //    the recording.
 109 class WebRtcAudioQualityBrowserTest : public WebRtcTestBase {
 110  public:
 111   WebRtcAudioQualityBrowserTest() {}
 112   virtual void SetUpInProcessBrowserTestFixture() override {
 113     DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
 114   }
 115
 116   virtual void SetUpCommandLine(CommandLine* command_line) override {
 117     // This test expects real device handling and requires a real webcam / audio
 118     // device; it will not work with fake devices.
 119     EXPECT_FALSE(command_line->HasSwitch(
 120         switches::kUseFakeDeviceForMediaStream));
 121     EXPECT_FALSE(command_line->HasSwitch(
 122         switches::kUseFakeUIForMediaStream));
 123   }
 124
 125   void AddAudioFile(const std::string& input_file_relative_url,
 126                     content::WebContents* tab_contents) {
 127     EXPECT_EQ("ok-added", ExecuteJavascript(
 128         "addAudioFile('" + input_file_relative_url + "')", tab_contents));
 129   }
 130
 131   void PlayAudioFile(content::WebContents* tab_contents) {
 132     EXPECT_EQ("ok-playing", ExecuteJavascript("playAudioFile()", tab_contents));
 133   }
 134
 135   base::FilePath CreateTemporaryWaveFile() {
 136     base::FilePath filename;
 137     EXPECT_TRUE(base::CreateTemporaryFile(&filename));
 138     base::FilePath wav_filename =
 139         filename.AddExtension(FILE_PATH_LITERAL(".wav"));
 140     EXPECT_TRUE(base::Move(filename, wav_filename));
 141     return wav_filename;
 142   }
 143 };
 144
 145 class AudioRecorder {
 146  public:
 147   AudioRecorder(): recording_application_(base::kNullProcessHandle) {}
 148   ~AudioRecorder() {}
 149
 150   // Starts the recording program for the specified duration. Returns true
 151   // on success.
 152   bool StartRecording(int duration_sec, const base::FilePath& output_file,
 153                       bool mono) {
 154     EXPECT_EQ(base::kNullProcessHandle, recording_application_)
 155         << "Tried to record, but is already recording.";
 156
 157     CommandLine command_line(CommandLine::NO_PROGRAM);
 158 #if defined(OS_WIN)
 159     // This disable is required to run SoundRecorder.exe on 64-bit Windows
 160     // from a 32-bit binary. We need to load the wow64 disable function from
 161     // the DLL since it doesn't exist on Windows XP.
 162     // TODO(phoglund): find some cleaner solution than using SoundRecorder.exe.
 163     base::ScopedNativeLibrary kernel32_lib(base::FilePath(L"kernel32"));
 164     if (kernel32_lib.is_valid()) {
 165       typedef BOOL (WINAPI* Wow64DisableWow64FSRedirection)(PVOID*);
 166       Wow64DisableWow64FSRedirection wow_64_disable_wow_64_fs_redirection;
 167       wow_64_disable_wow_64_fs_redirection =
 168           reinterpret_cast<Wow64DisableWow64FSRedirection>(
 169               kernel32_lib.GetFunctionPointer(
 170                   "Wow64DisableWow64FsRedirection"));
 171       if (wow_64_disable_wow_64_fs_redirection != NULL) {
 172         PVOID* ignored = NULL;
 173         wow_64_disable_wow_64_fs_redirection(ignored);
 174       }
 175     }
 176
 177     char duration_in_hms[128] = {0};
 178     struct tm duration_tm = {0};
 179     duration_tm.tm_sec = duration_sec;
 180     EXPECT_NE(0u, strftime(duration_in_hms, arraysize(duration_in_hms),
 181                            "%H:%M:%S", &duration_tm));
 182
 183     command_line.SetProgram(
 184         base::FilePath(FILE_PATH_LITERAL("SoundRecorder.exe")));
 185     command_line.AppendArg("/FILE");
 186     command_line.AppendArgPath(output_file);
 187     command_line.AppendArg("/DURATION");
 188     command_line.AppendArg(duration_in_hms);
 189 #elif defined(OS_MACOSX)
 190     command_line.SetProgram(base::FilePath("rec"));
 191     command_line.AppendArg("-b");
 192     command_line.AppendArg("16");
 193     command_line.AppendArg("-q");
 194     command_line.AppendArgPath(output_file);
 195     command_line.AppendArg("trim");
 196     command_line.AppendArg("0");
 197     command_line.AppendArg(base::StringPrintf("%d", duration_sec));
 198     command_line.AppendArg("rate");
 199     command_line.AppendArg("16k");
 200     if (mono) {
 201       command_line.AppendArg("remix");
 202       command_line.AppendArg("-");
 203     }
 204 #else
 205     int num_channels = mono ? 1 : 2;
 206     command_line.SetProgram(base::FilePath("arecord"));
 207     command_line.AppendArg("-d");
 208     command_line.AppendArg(base::StringPrintf("%d", duration_sec));
 209     command_line.AppendArg("-f");
 210     command_line.AppendArg("dat");
 211     command_line.AppendArg("-c");
 212     command_line.AppendArg(base::StringPrintf("%d", num_channels));
 213     command_line.AppendArgPath(output_file);
 214 #endif
 215
 216     VLOG(0) << "Running " << command_line.GetCommandLineString();
 217     return base::LaunchProcess(command_line, base::LaunchOptions(),
 218                                &recording_application_);
 219   }
 220
 221   // Joins the recording program. Returns true on success.
 222   bool WaitForRecordingToEnd() {
 223     int exit_code = -1;
 224     base::WaitForExitCode(recording_application_, &exit_code);
 225     return exit_code == 0;
 226   }
 227  private:
 228   base::ProcessHandle recording_application_;
 229 };
 230
 231 bool ForceMicrophoneVolumeTo100Percent() {
 232 #if defined(OS_WIN)
 233   // Note: the force binary isn't in tools since it's one of our own.
 234   CommandLine command_line(test::GetReferenceFilesDir().Append(
 235       FILE_PATH_LITERAL("force_mic_volume_max.exe")));
 236   VLOG(0) << "Running " << command_line.GetCommandLineString();
 237   std::string result;
 238   if (!base::GetAppOutput(command_line, &result)) {
 239     LOG(ERROR) << "Failed to set source volume: output was " << result;
 240     return false;
 241   }
 242 #elif defined(OS_MACOSX)
 243   CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("osascript")));
 244   command_line.AppendArg("-e");
 245   command_line.AppendArg("set volume input volume 100");
 246   command_line.AppendArg("-e");
 247   command_line.AppendArg("set volume output volume 100");
 248
 249   std::string result;
 250   if (!base::GetAppOutput(command_line, &result)) {
 251     LOG(ERROR) << "Failed to set source volume: output was " << result;
 252     return false;
 253   }
 254 #else
 255   // Just force the volume of, say the first 5 devices. A machine will rarely
 256   // have more input sources than that. This is way easier than finding the
 257   // input device we happen to be using.
 258   for (int device_index = 0; device_index < 5; ++device_index) {
 259     std::string result;
 260     const std::string kHundredPercentVolume = "65536";
 261     CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("pacmd")));
 262     command_line.AppendArg("set-source-volume");
 263     command_line.AppendArg(base::StringPrintf("%d", device_index));
 264     command_line.AppendArg(kHundredPercentVolume);
 265     VLOG(0) << "Running " << command_line.GetCommandLineString();
 266     if (!base::GetAppOutput(command_line, &result)) {
 267       LOG(ERROR) << "Failed to set source volume: output was " << result;
 268       return false;
 269     }
 270   }
 271 #endif
 272   return true;
 273 }
 274
 275 // Removes silence from beginning and end of the |input_audio_file| and writes
 276 // the result to the |output_audio_file|. Returns true on success.
 277 bool RemoveSilence(const base::FilePath& input_file,
 278                    const base::FilePath& output_file) {
 279   // SOX documentation for silence command: http://sox.sourceforge.net/sox.html
 280   // To remove the silence from both beginning and end of the audio file, we
 281   // call sox silence command twice: once on normal file and again on its
 282   // reverse, then we reverse the final output.
 283   // Silence parameters are (in sequence):
 284   // ABOVE_PERIODS: The period for which silence occurs. Value 1 is used for
 285   //                 silence at beginning of audio.
 286   // DURATION: the amount of time in seconds that non-silence must be detected
 287   //           before sox stops trimming audio.
 288   // THRESHOLD: value used to indicate what sample value is treates as silence.
 289   const char* kAbovePeriods = "1";
 290   const char* kDuration = "2";
 291   const char* kTreshold = "5%";
 292
 293 #if defined(OS_WIN)
 294   base::FilePath sox_path = test::GetReferenceFilesDir().Append(
 295       FILE_PATH_LITERAL("tools/sox.exe"));
 296   if (!base::PathExists(sox_path)) {
 297     LOG(ERROR) << "Missing sox.exe binary in " << sox_path.value()
 298                << "; you may have to provide this binary yourself.";
 299     return false;
 300   }
 301   CommandLine command_line(sox_path);
 302 #else
 303   CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("sox")));
 304 #endif
 305   command_line.AppendArgPath(input_file);
 306   command_line.AppendArgPath(output_file);
 307   command_line.AppendArg("silence");
 308   command_line.AppendArg(kAbovePeriods);
 309   command_line.AppendArg(kDuration);
 310   command_line.AppendArg(kTreshold);
 311   command_line.AppendArg("reverse");
 312   command_line.AppendArg("silence");
 313   command_line.AppendArg(kAbovePeriods);
 314   command_line.AppendArg(kDuration);
 315   command_line.AppendArg(kTreshold);
 316   command_line.AppendArg("reverse");
 317
 318   VLOG(0) << "Running " << command_line.GetCommandLineString();
 319   std::string result;
 320   bool ok = base::GetAppOutput(command_line, &result);
 321   VLOG(0) << "Output was:\n\n" << result;
 322   return ok;
 323 }
 324
 325 bool CanParseAsFloat(const std::string& value) {
 326   return atof(value.c_str()) != 0 || value == "0";
 327 }
 328
 329 // Runs PESQ to compare |reference_file| to a |actual_file|. The |sample_rate|
 330 // can be either 16000 or 8000.
 331 //
 332 // PESQ is only mono-aware, so the files should preferably be recorded in mono.
 333 // Furthermore it expects the file to be 16 rather than 32 bits, even though
 334 // 32 bits might work. The audio bandwidth of the two files should be the same
 335 // e.g. don't compare a 32 kHz file to a 8 kHz file.
 336 //
 337 // The raw score in MOS is written to |raw_mos|, whereas the MOS-LQO score is
 338 // written to mos_lqo. The scores are returned as floats in string form (e.g.
 339 // "3.145", etc). Returns true on success.
 340 bool RunPesq(const base::FilePath& reference_file,
 341              const base::FilePath& actual_file,
 342              int sample_rate, std::string* raw_mos, std::string* mos_lqo) {
 343   // PESQ will break if the paths are too long (!).
 344   EXPECT_LT(reference_file.value().length(), 128u);
 345   EXPECT_LT(actual_file.value().length(), 128u);
 346
 347 #if defined(OS_WIN)
 348   base::FilePath pesq_path =
 349       test::GetReferenceFilesDir().Append(FILE_PATH_LITERAL("tools/pesq.exe"));
 350 #elif defined(OS_MACOSX)
 351   base::FilePath pesq_path =
 352       test::GetReferenceFilesDir().Append(FILE_PATH_LITERAL("tools/pesq_mac"));
 353 #else
 354   base::FilePath pesq_path =
 355       test::GetReferenceFilesDir().Append(FILE_PATH_LITERAL("tools/pesq"));
 356 #endif
 357
 358   if (!base::PathExists(pesq_path)) {
 359     LOG(ERROR) << "Missing PESQ binary in " << pesq_path.value()
 360                << "; you may have to provide this binary yourself.";
 361     return false;
 362   }
 363
 364   CommandLine command_line(pesq_path);
 365   command_line.AppendArg(base::StringPrintf("+%d", sample_rate));
 366   command_line.AppendArgPath(reference_file);
 367   command_line.AppendArgPath(actual_file);
 368
 369   VLOG(0) << "Running " << command_line.GetCommandLineString();
 370   std::string result;
 371   if (!base::GetAppOutput(command_line, &result)) {
 372     LOG(ERROR) << "Failed to run PESQ.";
 373     return false;
 374   }
 375   VLOG(0) << "Output was:\n\n" << result;
 376
 377   const std::string result_anchor = "Prediction (Raw MOS, MOS-LQO):  = ";
 378   std::size_t anchor_pos = result.find(result_anchor);
 379   if (anchor_pos == std::string::npos) {
 380     LOG(ERROR) << "PESQ was not able to compute a score; we probably recorded "
 381         << "only silence. Please check the output/input volume levels.";
 382     return false;
 383   }
 384
 385   // There are two tab-separated numbers on the format x.xxx, e.g. 5 chars each.
 386   std::size_t first_number_pos = anchor_pos + result_anchor.length();
 387   *raw_mos = result.substr(first_number_pos, 5);
 388   EXPECT_TRUE(CanParseAsFloat(*raw_mos)) << "Failed to parse raw MOS number.";
 389   *mos_lqo = result.substr(first_number_pos + 5 + 1, 5);
 390   EXPECT_TRUE(CanParseAsFloat(*mos_lqo)) << "Failed to parse MOS LQO number.";
 391
 392   return true;
 393 }
 394
 395 #if defined(OS_LINUX) || defined(OS_WIN) || defined(OS_MACOSX)
 396 #define MAYBE_MANUAL_TestAudioQuality MANUAL_TestAudioQuality
 397 #else
 398 // Not implemented on Android, ChromeOS etc.
 399 #define MAYBE_MANUAL_TestAudioQuality DISABLED_MANUAL_TestAudioQuality
 400 #endif
 401
 402 IN_PROC_BROWSER_TEST_F(WebRtcAudioQualityBrowserTest,
 403                        MAYBE_MANUAL_TestAudioQuality) {
 404   if (OnWinXp()) {
 405     LOG(ERROR) << "This test is not implemented for Windows XP.";
 406     return;
 407   }
 408   if (OnWin8()) {
 409     // http://crbug.com/379798.
 410     LOG(ERROR) << "Temporarily disabled for Win 8.";
 411     return;
 412   }
 413   ASSERT_TRUE(test::HasReferenceFilesInCheckout());
 414   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 415
 416   ASSERT_TRUE(ForceMicrophoneVolumeTo100Percent());
 417
 418   ui_test_utils::NavigateToURL(
 419       browser(), embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage));
 420   content::WebContents* left_tab =
 421       browser()->tab_strip_model()->GetActiveWebContents();
 422
 423   chrome::AddTabAt(browser(), GURL(), -1, true);
 424   content::WebContents* right_tab =
 425       browser()->tab_strip_model()->GetActiveWebContents();
 426   ui_test_utils::NavigateToURL(
 427       browser(), embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage));
 428
 429   // Prepare the peer connections manually in this test since we don't add
 430   // getUserMedia-derived media streams in this test like the other tests.
 431   EXPECT_EQ("ok-peerconnection-created",
 432             ExecuteJavascript("preparePeerConnection()", left_tab));
 433   EXPECT_EQ("ok-peerconnection-created",
 434             ExecuteJavascript("preparePeerConnection()", right_tab));
 435
 436   AddAudioFile(kReferenceFileRelativeUrl, left_tab);
 437
 438   NegotiateCall(left_tab, right_tab);
 439
 440   // Note: the media flow isn't necessarily established on the connection just
 441   // because the ready state is ok on both sides. We sleep a bit between call
 442   // establishment and playing to avoid cutting of the beginning of the audio
 443   // file.
 444   test::SleepInJavascript(left_tab, 2000);
 445
 446   base::FilePath recording = CreateTemporaryWaveFile();
 447
 448   // Note: the sound clip is about 10 seconds: record for 15 seconds to get some
 449   // safety margins on each side.
 450   AudioRecorder recorder;
 451   static int kRecordingTimeSeconds = 15;
 452   ASSERT_TRUE(recorder.StartRecording(kRecordingTimeSeconds, recording, true));
 453
 454   PlayAudioFile(left_tab);
 455
 456   ASSERT_TRUE(recorder.WaitForRecordingToEnd());
 457   VLOG(0) << "Done recording to " << recording.value() << std::endl;
 458
 459   HangUp(left_tab);
 460
 461   base::FilePath trimmed_recording = CreateTemporaryWaveFile();
 462
 463   ASSERT_TRUE(RemoveSilence(recording, trimmed_recording));
 464   VLOG(0) << "Trimmed silence: " << trimmed_recording.value() << std::endl;
 465
 466   std::string raw_mos;
 467   std::string mos_lqo;
 468   base::FilePath reference_file_in_test_dir =
 469       test::GetReferenceFilesDir().Append(kReferenceFile);
 470   ASSERT_TRUE(RunPesq(reference_file_in_test_dir, trimmed_recording, 16000,
 471                       &raw_mos, &mos_lqo));
 472
 473   perf_test::PrintResult("audio_pesq", "", "raw_mos", raw_mos, "score", true);
 474   perf_test::PrintResult("audio_pesq", "", "mos_lqo", mos_lqo, "score", true);
 475
 476   EXPECT_TRUE(base::DeleteFile(recording, false));
 477   EXPECT_TRUE(base::DeleteFile(trimmed_recording, false));
 478 }