media/ffmpeg/ffmpeg_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 // ffmpeg_unittests verify that the parts of the FFmpeg API that Chromium uses
   6 // function as advertised for each media format that Chromium supports.  This
   7 // mostly includes stuff like reporting proper timestamps, seeking to
   8 // keyframes, and supporting certain features like reordered_opaque.
   9 //
  10
  11 #include <limits>
  12 #include <queue>
  13
  14 #include "base/base_paths.h"
  15 #include "base/file_path.h"
  16 #include "base/file_util.h"
  17 #include "base/path_service.h"
  18 #include "base/perftimer.h"
  19 #include "base/string_util.h"
  20 #include "base/test/perf_test_suite.h"
  21 #include "media/base/media.h"
  22 #include "media/ffmpeg/ffmpeg_common.h"
  23 #include "media/ffmpeg/file_protocol.h"
  24 #include "testing/gtest/include/gtest/gtest.h"
  25
  26 int main(int argc, char** argv) {
  27   return base::PerfTestSuite(argc, argv).Run();
  28 }
  29
  30 namespace media {
  31
  32 // Mirror setting in ffmpeg_video_decoder.
  33 static const int kDecodeThreads = 2;
  34
  35 class AVPacketQueue {
  36  public:
  37   AVPacketQueue() {
  38   }
  39
  40   ~AVPacketQueue() {
  41     flush();
  42   }
  43
  44   bool empty() {
  45     return packets_.empty();
  46   }
  47
  48   AVPacket* peek() {
  49     return packets_.front();
  50   }
  51
  52   void pop() {
  53     AVPacket* packet = packets_.front();
  54     packets_.pop();
  55     av_free_packet(packet);
  56     delete packet;
  57   }
  58
  59   void push(AVPacket* packet) {
  60     av_dup_packet(packet);
  61     packets_.push(packet);
  62   }
  63
  64   void flush() {
  65     while (!empty()) {
  66       pop();
  67     }
  68   }
  69
  70  private:
  71   std::queue<AVPacket*> packets_;
  72
  73   DISALLOW_COPY_AND_ASSIGN(AVPacketQueue);
  74 };
  75
  76 // TODO(dalecurtis): We should really just use PipelineIntegrationTests instead
  77 // of a one-off step decoder so we're exercising the real pipeline.
  78 class FFmpegTest : public testing::TestWithParam<const char*> {
  79  protected:
  80   FFmpegTest()
  81       : av_format_context_(NULL),
  82         audio_stream_index_(-1),
  83         video_stream_index_(-1),
  84         audio_buffer_(NULL),
  85         video_buffer_(NULL),
  86         decoded_audio_time_(AV_NOPTS_VALUE),
  87         decoded_audio_duration_(AV_NOPTS_VALUE),
  88         decoded_video_time_(AV_NOPTS_VALUE),
  89         decoded_video_duration_(AV_NOPTS_VALUE),
  90         duration_(AV_NOPTS_VALUE) {
  91     InitializeFFmpeg();
  92
  93     audio_buffer_.reset(avcodec_alloc_frame());
  94     video_buffer_.reset(avcodec_alloc_frame());
  95   }
  96
  97   virtual ~FFmpegTest() {
  98   }
  99
 100   void OpenAndReadFile(const std::string& name) {
 101     OpenFile(name);
 102     OpenCodecs();
 103     ReadRemainingFile();
 104   }
 105
 106   void OpenFile(const std::string& name) {
 107     FilePath path;
 108     PathService::Get(base::DIR_SOURCE_ROOT, &path);
 109     path = path.AppendASCII("media")
 110         .AppendASCII("test")
 111         .AppendASCII("data")
 112         .AppendASCII("content")
 113         .AppendASCII(name.c_str());
 114     FilePath::StringType raw_path = path.value();
 115     EXPECT_TRUE(file_util::PathExists(path));
 116
 117 #if defined(OS_WIN)
 118     std::string ascii_path = WideToASCII(path.value());
 119 #else
 120     std::string ascii_path = path.value();
 121 #endif
 122
 123     EXPECT_EQ(0, avformat_open_input(&av_format_context_,
 124                                      ascii_path.c_str(),
 125                                      NULL, NULL))
 126         << "Could not open " << path.value();
 127     EXPECT_LE(0, avformat_find_stream_info(av_format_context_, NULL))
 128         << "Could not find stream information for " << path.value();
 129
 130     // Determine duration by picking max stream duration.
 131     for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
 132       AVStream* av_stream = av_format_context_->streams[i];
 133       int64 duration = ConvertFromTimeBase(
 134           av_stream->time_base, av_stream->duration).InMicroseconds();
 135       duration_ = std::max(duration_, duration);
 136     }
 137
 138     // Final check to see if the container itself specifies a duration.
 139     AVRational av_time_base = {1, AV_TIME_BASE};
 140     int64 duration =
 141         ConvertFromTimeBase(av_time_base,
 142                             av_format_context_->duration).InMicroseconds();
 143     duration_ = std::max(duration_, duration);
 144   }
 145
 146   void CloseFile() {
 147     avformat_close_input(&av_format_context_);
 148   }
 149
 150   void OpenCodecs() {
 151     for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
 152       AVStream* av_stream = av_format_context_->streams[i];
 153       AVCodecContext* av_codec_context = av_stream->codec;
 154       AVCodec* av_codec = avcodec_find_decoder(av_codec_context->codec_id);
 155
 156       EXPECT_TRUE(av_codec)
 157           << "Could not find AVCodec with CodecID "
 158           << av_codec_context->codec_id;
 159
 160       av_codec_context->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
 161       av_codec_context->err_recognition = AV_EF_CAREFUL;
 162       av_codec_context->thread_count = kDecodeThreads;
 163
 164       EXPECT_EQ(0, avcodec_open2(av_codec_context, av_codec, NULL))
 165           << "Could not open AVCodecContext with CodecID "
 166           << av_codec_context->codec_id;
 167
 168       if (av_codec->type == AVMEDIA_TYPE_AUDIO) {
 169         EXPECT_EQ(-1, audio_stream_index_) << "Found multiple audio streams.";
 170         audio_stream_index_ = static_cast<int>(i);
 171       } else if (av_codec->type == AVMEDIA_TYPE_VIDEO) {
 172         EXPECT_EQ(-1, video_stream_index_) << "Found multiple video streams.";
 173         video_stream_index_ = static_cast<int>(i);
 174       } else {
 175         ADD_FAILURE() << "Found unknown stream type.";
 176       }
 177     }
 178   }
 179
 180   void CloseCodecs() {
 181     for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
 182       AVStream* av_stream = av_format_context_->streams[i];
 183       av_stream->discard = AVDISCARD_ALL;
 184       avcodec_close(av_stream->codec);
 185     }
 186   }
 187
 188   void Flush() {
 189     if (has_audio()) {
 190       audio_packets_.flush();
 191       avcodec_flush_buffers(av_audio_context());
 192     }
 193     if (has_video()) {
 194       video_packets_.flush();
 195       avcodec_flush_buffers(av_video_context());
 196     }
 197   }
 198
 199   void ReadUntil(int64 time) {
 200     while (true) {
 201       scoped_ptr<AVPacket> packet(new AVPacket());
 202       if (av_read_frame(av_format_context_, packet.get()) < 0) {
 203         break;
 204       }
 205
 206       int stream_index = static_cast<int>(packet->stream_index);
 207       int64 packet_time = AV_NOPTS_VALUE;
 208       if (stream_index == audio_stream_index_) {
 209         packet_time =
 210             ConvertFromTimeBase(av_audio_stream()->time_base, packet->pts)
 211                 .InMicroseconds();
 212         audio_packets_.push(packet.release());
 213       } else if (stream_index == video_stream_index_) {
 214         packet_time =
 215             ConvertFromTimeBase(av_video_stream()->time_base, packet->pts)
 216                 .InMicroseconds();
 217         video_packets_.push(packet.release());
 218       } else {
 219         ADD_FAILURE() << "Found packet that belongs to unknown stream.";
 220       }
 221
 222       if (packet_time > time) {
 223         break;
 224       }
 225     }
 226   }
 227
 228   void ReadRemainingFile() {
 229     ReadUntil(std::numeric_limits<int64>::max());
 230   }
 231
 232   bool StepDecodeAudio() {
 233     EXPECT_TRUE(has_audio());
 234     if (!has_audio() || audio_packets_.empty()) {
 235       return false;
 236     }
 237
 238     // Decode until output is produced, end of stream, or error.
 239     while (true) {
 240       int result = 0;
 241       int got_audio = 0;
 242       bool end_of_stream = false;
 243
 244       AVPacket packet;
 245       if (audio_packets_.empty()) {
 246         av_init_packet(&packet);
 247         end_of_stream = true;
 248       } else {
 249         memcpy(&packet, audio_packets_.peek(), sizeof(packet));
 250       }
 251
 252       avcodec_get_frame_defaults(audio_buffer_.get());
 253       result = avcodec_decode_audio4(av_audio_context(), audio_buffer_.get(),
 254                                      &got_audio, &packet);
 255       if (!audio_packets_.empty()) {
 256         audio_packets_.pop();
 257       }
 258
 259       EXPECT_GE(result, 0) << "Audio decode error.";
 260       if (result < 0 || (got_audio == 0 && end_of_stream)) {
 261         return false;
 262       }
 263
 264       if (result > 0) {
 265         double microseconds = 1.0L * audio_buffer_->nb_samples /
 266             av_audio_context()->sample_rate *
 267             base::Time::kMicrosecondsPerSecond;
 268         decoded_audio_duration_ = static_cast<int64>(microseconds);
 269
 270         if (packet.pts == static_cast<int64>(AV_NOPTS_VALUE)) {
 271           EXPECT_NE(decoded_audio_time_, static_cast<int64>(AV_NOPTS_VALUE))
 272               << "We never received an initial timestamped audio packet! "
 273               << "Looks like there's a seeking/parsing bug in FFmpeg.";
 274           decoded_audio_time_ += decoded_audio_duration_;
 275         } else {
 276           decoded_audio_time_ =
 277               ConvertFromTimeBase(av_audio_stream()->time_base, packet.pts)
 278                   .InMicroseconds();
 279         }
 280         return true;
 281       }
 282     }
 283     return true;
 284   }
 285
 286   bool StepDecodeVideo() {
 287     EXPECT_TRUE(has_video());
 288     if (!has_video() || video_packets_.empty()) {
 289       return false;
 290     }
 291
 292     // Decode until output is produced, end of stream, or error.
 293     while (true) {
 294       int result = 0;
 295       int got_picture = 0;
 296       bool end_of_stream = false;
 297
 298       AVPacket packet;
 299       if (video_packets_.empty()) {
 300         av_init_packet(&packet);
 301         end_of_stream = true;
 302       } else {
 303         memcpy(&packet, video_packets_.peek(), sizeof(packet));
 304       }
 305
 306       avcodec_get_frame_defaults(video_buffer_.get());
 307       av_video_context()->reordered_opaque = packet.pts;
 308       result = avcodec_decode_video2(av_video_context(), video_buffer_.get(),
 309                                      &got_picture, &packet);
 310       if (!video_packets_.empty()) {
 311         video_packets_.pop();
 312       }
 313
 314       EXPECT_GE(result, 0) << "Video decode error.";
 315       if (result < 0 || (got_picture == 0 && end_of_stream)) {
 316         return false;
 317       }
 318
 319       if (got_picture) {
 320         AVRational doubled_time_base;
 321         doubled_time_base.den = av_video_stream()->r_frame_rate.num;
 322         doubled_time_base.num = av_video_stream()->r_frame_rate.den;
 323         doubled_time_base.den *= 2;
 324
 325         decoded_video_time_ =
 326             ConvertFromTimeBase(av_video_stream()->time_base,
 327                              video_buffer_->reordered_opaque)
 328                 .InMicroseconds();
 329         decoded_video_duration_ =
 330             ConvertFromTimeBase(doubled_time_base,
 331                              2 + video_buffer_->repeat_pict)
 332                 .InMicroseconds();
 333         return true;
 334       }
 335     }
 336   }
 337
 338   void DecodeRemainingAudio() {
 339     while (StepDecodeAudio()) {}
 340   }
 341
 342   void DecodeRemainingVideo() {
 343     while (StepDecodeVideo()) {}
 344   }
 345
 346   void SeekTo(double position) {
 347     int64 seek_time =
 348         static_cast<int64>(position * base::Time::kMicrosecondsPerSecond);
 349     int flags = AVSEEK_FLAG_BACKWARD;
 350
 351     // Passing -1 as our stream index lets FFmpeg pick a default stream.
 352     // FFmpeg will attempt to use the lowest-index video stream, if present,
 353     // followed by the lowest-index audio stream.
 354     EXPECT_GE(0, av_seek_frame(av_format_context_, -1, seek_time, flags))
 355         << "Failed to seek to position " << position;
 356     Flush();
 357   }
 358
 359   bool has_audio() { return audio_stream_index_ >= 0; }
 360   bool has_video() { return video_stream_index_ >= 0; }
 361   int64 decoded_audio_time() { return decoded_audio_time_; }
 362   int64 decoded_audio_duration() { return decoded_audio_duration_; }
 363   int64 decoded_video_time() { return decoded_video_time_; }
 364   int64 decoded_video_duration() { return decoded_video_duration_; }
 365   int64 duration() { return duration_; }
 366
 367   AVStream* av_audio_stream() {
 368     return av_format_context_->streams[audio_stream_index_];
 369   }
 370   AVStream* av_video_stream() {
 371     return av_format_context_->streams[video_stream_index_];
 372   }
 373   AVCodecContext* av_audio_context() {
 374     return av_audio_stream()->codec;
 375   }
 376   AVCodecContext* av_video_context() {
 377     return av_video_stream()->codec;
 378   }
 379
 380  private:
 381   void InitializeFFmpeg() {
 382     static bool initialized = false;
 383     if (initialized) {
 384       return;
 385     }
 386
 387     FilePath path;
 388     PathService::Get(base::DIR_MODULE, &path);
 389     EXPECT_TRUE(InitializeMediaLibrary(path))
 390         << "Could not initialize media library.";
 391
 392     av_log_set_level(AV_LOG_FATAL);
 393     av_register_all();
 394     av_register_protocol2(&kFFmpegFileProtocol, sizeof(kFFmpegFileProtocol));
 395     initialized = true;
 396   }
 397
 398   AVFormatContext* av_format_context_;
 399   int audio_stream_index_;
 400   int video_stream_index_;
 401   AVPacketQueue audio_packets_;
 402   AVPacketQueue video_packets_;
 403
 404   scoped_ptr_malloc<AVFrame, media::ScopedPtrAVFree> audio_buffer_;
 405   scoped_ptr_malloc<AVFrame, media::ScopedPtrAVFree> video_buffer_;
 406
 407   int64 decoded_audio_time_;
 408   int64 decoded_audio_duration_;
 409   int64 decoded_video_time_;
 410   int64 decoded_video_duration_;
 411   int64 duration_;
 412
 413   DISALLOW_COPY_AND_ASSIGN(FFmpegTest);
 414 };
 415
 416 #define FFMPEG_TEST_CASE(name, extension) \
 417     INSTANTIATE_TEST_CASE_P(name##_##extension, FFmpegTest, \
 418                             testing::Values(#name "." #extension));
 419
 420 // Covers all our basic formats.
 421 FFMPEG_TEST_CASE(sync0, mp4);
 422 FFMPEG_TEST_CASE(sync0, ogv);
 423 FFMPEG_TEST_CASE(sync0, webm);
 424 FFMPEG_TEST_CASE(sync1, m4a);
 425 FFMPEG_TEST_CASE(sync1, mp3);
 426 FFMPEG_TEST_CASE(sync1, mp4);
 427 FFMPEG_TEST_CASE(sync1, ogg);
 428 FFMPEG_TEST_CASE(sync1, ogv);
 429 FFMPEG_TEST_CASE(sync1, webm);
 430 FFMPEG_TEST_CASE(sync2, m4a);
 431 FFMPEG_TEST_CASE(sync2, mp3);
 432 FFMPEG_TEST_CASE(sync2, mp4);
 433 FFMPEG_TEST_CASE(sync2, ogg);
 434 FFMPEG_TEST_CASE(sync2, ogv);
 435 FFMPEG_TEST_CASE(sync2, webm);
 436
 437 // Covers our LayoutTest file.
 438 FFMPEG_TEST_CASE(counting, ogv);
 439
 440 TEST_P(FFmpegTest, Perf) {
 441   {
 442     PerfTimeLogger timer("Opening file");
 443     OpenFile(GetParam());
 444   }
 445   {
 446     PerfTimeLogger timer("Opening codecs");
 447     OpenCodecs();
 448   }
 449   {
 450     PerfTimeLogger timer("Reading file");
 451     ReadRemainingFile();
 452   }
 453   if (has_audio()) {
 454     PerfTimeLogger timer("Decoding audio");
 455     DecodeRemainingAudio();
 456   }
 457   if (has_video()) {
 458     PerfTimeLogger timer("Decoding video");
 459     DecodeRemainingVideo();
 460   }
 461   {
 462     PerfTimeLogger timer("Seeking to zero");
 463     SeekTo(0);
 464   }
 465   {
 466     PerfTimeLogger timer("Closing codecs");
 467     CloseCodecs();
 468   }
 469   {
 470     PerfTimeLogger timer("Closing file");
 471     CloseFile();
 472   }
 473 }
 474
 475 TEST_P(FFmpegTest, Loop_Audio) {
 476   OpenAndReadFile(GetParam());
 477   if (!has_audio()) {
 478     return;
 479   }
 480
 481   const int kSteps = 4;
 482   std::vector<int64> expected_timestamps_;
 483   for (int i = 0; i < kSteps; ++i) {
 484     EXPECT_TRUE(StepDecodeAudio());
 485     expected_timestamps_.push_back(decoded_audio_time());
 486   }
 487
 488   SeekTo(0);
 489   ReadRemainingFile();
 490
 491   for (int i = 0; i < kSteps; ++i) {
 492     EXPECT_TRUE(StepDecodeAudio());
 493     EXPECT_EQ(expected_timestamps_[i], decoded_audio_time())
 494         << "Frame " << i << " had a mismatched timestamp.";
 495   }
 496
 497   CloseCodecs();
 498   CloseFile();
 499 }
 500
 501 TEST_P(FFmpegTest, Loop_Video) {
 502   OpenAndReadFile(GetParam());
 503   if (!has_video()) {
 504     return;
 505   }
 506
 507   const int kSteps = 4;
 508   std::vector<int64> expected_timestamps_;
 509   for (int i = 0; i < kSteps; ++i) {
 510     EXPECT_TRUE(StepDecodeVideo());
 511     expected_timestamps_.push_back(decoded_video_time());
 512   }
 513
 514   SeekTo(0);
 515   ReadRemainingFile();
 516
 517   for (int i = 0; i < kSteps; ++i) {
 518     EXPECT_TRUE(StepDecodeVideo());
 519     EXPECT_EQ(expected_timestamps_[i], decoded_video_time())
 520         << "Frame " << i << " had a mismatched timestamp.";
 521   }
 522
 523   CloseCodecs();
 524   CloseFile();
 525 }
 526
 527 TEST_P(FFmpegTest, Seek_Audio) {
 528   OpenAndReadFile(GetParam());
 529   if (!has_audio() && duration() >= 0.5) {
 530     return;
 531   }
 532
 533   SeekTo(duration() - 0.5);
 534   ReadRemainingFile();
 535
 536   EXPECT_TRUE(StepDecodeAudio());
 537   EXPECT_NE(static_cast<int64>(AV_NOPTS_VALUE), decoded_audio_time());
 538
 539   CloseCodecs();
 540   CloseFile();
 541 }
 542
 543 TEST_P(FFmpegTest, Seek_Video) {
 544   OpenAndReadFile(GetParam());
 545   if (!has_video() && duration() >= 0.5) {
 546     return;
 547   }
 548
 549   SeekTo(duration() - 0.5);
 550   ReadRemainingFile();
 551
 552   EXPECT_TRUE(StepDecodeVideo());
 553   EXPECT_NE(static_cast<int64>(AV_NOPTS_VALUE), decoded_video_time());
 554
 555   CloseCodecs();
 556   CloseFile();
 557 }
 558
 559 TEST_P(FFmpegTest, Decode_Audio) {
 560   OpenAndReadFile(GetParam());
 561   if (!has_audio()) {
 562     return;
 563   }
 564
 565   int64 last_audio_time = AV_NOPTS_VALUE;
 566   while (StepDecodeAudio()) {
 567     ASSERT_GT(decoded_audio_time(), last_audio_time);
 568     last_audio_time = decoded_audio_time();
 569   }
 570
 571   CloseCodecs();
 572   CloseFile();
 573 }
 574
 575 TEST_P(FFmpegTest, Decode_Video) {
 576   OpenAndReadFile(GetParam());
 577   if (!has_video()) {
 578     return;
 579   }
 580
 581   int64 last_video_time = AV_NOPTS_VALUE;
 582   while (StepDecodeVideo()) {
 583     ASSERT_GT(decoded_video_time(), last_video_time);
 584     last_video_time = decoded_video_time();
 585   }
 586
 587   CloseCodecs();
 588   CloseFile();
 589 }
 590
 591 TEST_P(FFmpegTest, Duration) {
 592   OpenAndReadFile(GetParam());
 593
 594   if (has_audio()) {
 595     DecodeRemainingAudio();
 596   }
 597
 598   if (has_video()) {
 599     DecodeRemainingVideo();
 600   }
 601
 602   double expected = static_cast<double>(duration());
 603   double actual = static_cast<double>(
 604       std::max(decoded_audio_time() + decoded_audio_duration(),
 605                decoded_video_time() + decoded_video_duration()));
 606   EXPECT_NEAR(expected, actual, 500000)
 607       << "Duration is off by more than 0.5 seconds.";
 608
 609   CloseCodecs();
 610   CloseFile();
 611 }
 612
 613 TEST_F(FFmpegTest, VideoPlayedCollapse) {
 614   OpenFile("test.ogv");
 615   OpenCodecs();
 616
 617   SeekTo(0.5);
 618   ReadRemainingFile();
 619   EXPECT_TRUE(StepDecodeVideo());
 620   VLOG(1) << decoded_video_time();
 621
 622   SeekTo(2.83);
 623   ReadRemainingFile();
 624   EXPECT_TRUE(StepDecodeVideo());
 625   VLOG(1) << decoded_video_time();
 626
 627   SeekTo(0.4);
 628   ReadRemainingFile();
 629   EXPECT_TRUE(StepDecodeVideo());
 630   VLOG(1) << decoded_video_time();
 631
 632   CloseCodecs();
 633   CloseFile();
 634 }
 635
 636 }  // namespace media