media/cast/sender/video_encoder_unittest.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <vector>
   6 #include <utility>
   7
   8 #include "base/bind.h"
   9 #include "base/memory/ref_counted.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "media/base/video_frame.h"
  12 #include "media/cast/cast_defines.h"
  13 #include "media/cast/cast_environment.h"
  14 #include "media/cast/sender/fake_video_encode_accelerator_factory.h"
  15 #include "media/cast/sender/video_frame_factory.h"
  16 #include "media/cast/sender/video_encoder.h"
  17 #include "media/cast/test/fake_single_thread_task_runner.h"
  18 #include "media/cast/test/utility/default_config.h"
  19 #include "media/cast/test/utility/video_utility.h"
  20 #include "testing/gtest/include/gtest/gtest.h"
  21
  22 #if defined(OS_MACOSX)
  23 #include "media/cast/sender/h264_vt_encoder.h"
  24 #endif
  25
  26 namespace media {
  27 namespace cast {
  28
  29 class VideoEncoderTest
  30     : public ::testing::TestWithParam<std::pair<Codec, bool>> {
  31  protected:
  32   VideoEncoderTest()
  33       : testing_clock_(new base::SimpleTestTickClock()),
  34         task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)),
  35         cast_environment_(new CastEnvironment(
  36             scoped_ptr<base::TickClock>(testing_clock_).Pass(),
  37             task_runner_,
  38             task_runner_,
  39             task_runner_)),
  40         video_config_(GetDefaultVideoSenderConfig()),
  41         operational_status_(STATUS_UNINITIALIZED),
  42         count_frames_delivered_(0) {
  43     testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks());
  44     first_frame_time_ = testing_clock_->NowTicks();
  45   }
  46
  47   ~VideoEncoderTest() override {}
  48
  49   void SetUp() final {
  50     video_config_.codec = GetParam().first;
  51     video_config_.use_external_encoder = GetParam().second;
  52
  53     if (video_config_.use_external_encoder)
  54       vea_factory_.reset(new FakeVideoEncodeAcceleratorFactory(task_runner_));
  55   }
  56
  57   void TearDown() final {
  58     video_encoder_.reset();
  59     RunTasksAndAdvanceClock();
  60   }
  61
  62   void CreateEncoder(bool three_buffer_mode) {
  63     ASSERT_EQ(STATUS_UNINITIALIZED, operational_status_);
  64     video_config_.max_number_of_video_buffers_used =
  65         (three_buffer_mode ? 3 : 1);
  66     video_encoder_ = VideoEncoder::Create(
  67         cast_environment_,
  68         video_config_,
  69         base::Bind(&VideoEncoderTest::OnOperationalStatusChange,
  70                    base::Unretained(this)),
  71         base::Bind(
  72             &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator,
  73             base::Unretained(vea_factory_.get())),
  74         base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory,
  75                    base::Unretained(vea_factory_.get()))).Pass();
  76     RunTasksAndAdvanceClock();
  77     if (is_encoder_present())
  78       ASSERT_EQ(STATUS_INITIALIZED, operational_status_);
  79   }
  80
  81   bool is_encoder_present() const {
  82     return !!video_encoder_;
  83   }
  84
  85   bool is_testing_software_vp8_encoder() const {
  86     return video_config_.codec == CODEC_VIDEO_VP8 &&
  87         !video_config_.use_external_encoder;
  88   }
  89
  90   bool is_testing_video_toolbox_encoder() const {
  91     return
  92 #if defined(OS_MACOSX)
  93         (!video_config_.use_external_encoder &&
  94          H264VideoToolboxEncoder::IsSupported(video_config_)) ||
  95 #endif
  96         false;
  97   }
  98
  99   bool is_testing_platform_encoder() const {
 100     return video_config_.use_external_encoder ||
 101         is_testing_video_toolbox_encoder();
 102   }
 103
 104   bool encoder_has_resize_delay() const {
 105     return is_testing_platform_encoder() && !is_testing_video_toolbox_encoder();
 106   }
 107
 108   VideoEncoder* video_encoder() const {
 109     return video_encoder_.get();
 110   }
 111
 112   void DestroyEncoder() {
 113     video_encoder_.reset();
 114   }
 115
 116   base::TimeTicks Now() const {
 117     return testing_clock_->NowTicks();
 118   }
 119
 120   void RunTasksAndAdvanceClock() const {
 121     const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds(
 122         1000000.0 / video_config_.max_frame_rate);
 123 #if defined(OS_MACOSX)
 124     if (is_testing_video_toolbox_encoder()) {
 125       // The H264VideoToolboxEncoder (on MAC_OSX and IOS) is not a faked
 126       // implementation in these tests, and performs its encoding asynchronously
 127       // on an unknown set of threads.  Therefore, sleep the current thread for
 128       // the real amount of time to avoid excessively spinning the CPU while
 129       // waiting for something to happen.
 130       base::PlatformThread::Sleep(frame_duration);
 131     }
 132 #endif
 133     task_runner_->RunTasks();
 134     testing_clock_->Advance(frame_duration);
 135   }
 136
 137   int count_frames_delivered() const {
 138     return count_frames_delivered_;
 139   }
 140
 141   void WaitForAllFramesToBeDelivered(int total_expected) const {
 142     video_encoder_->EmitFrames();
 143     while (count_frames_delivered_ < total_expected)
 144       RunTasksAndAdvanceClock();
 145   }
 146
 147   // Creates a new VideoFrame of the given |size|, filled with a test pattern.
 148   // When available, it attempts to use the VideoFrameFactory provided by the
 149   // encoder.
 150   scoped_refptr<media::VideoFrame> CreateTestVideoFrame(const gfx::Size& size) {
 151     const base::TimeDelta timestamp =
 152         testing_clock_->NowTicks() - first_frame_time_;
 153     scoped_refptr<media::VideoFrame> frame;
 154     if (video_frame_factory_)
 155       frame = video_frame_factory_->MaybeCreateFrame(size, timestamp);
 156     if (!frame) {
 157       frame = media::VideoFrame::CreateFrame(
 158           VideoFrame::I420, size, gfx::Rect(size), size, timestamp);
 159     }
 160     PopulateVideoFrame(frame.get(), 123);
 161     return frame;
 162   }
 163
 164   // Requests encoding the |video_frame| and has the resulting frame delivered
 165   // via a callback that checks for expected results.  Returns false if the
 166   // encoder rejected the request.
 167   bool EncodeAndCheckDelivery(
 168       const scoped_refptr<media::VideoFrame>& video_frame,
 169       uint32 frame_id,
 170       uint32 reference_frame_id) {
 171     return video_encoder_->EncodeVideoFrame(
 172         video_frame,
 173         Now(),
 174         base::Bind(&VideoEncoderTest::DeliverEncodedVideoFrame,
 175                    base::Unretained(this),
 176                    frame_id,
 177                    reference_frame_id,
 178                    TimeDeltaToRtpDelta(video_frame->timestamp(),
 179                                        kVideoFrequency),
 180                    Now()));
 181   }
 182
 183   // If the implementation of |video_encoder_| is ExternalVideoEncoder, check
 184   // that the VEA factory has responded (by running the callbacks) a specific
 185   // number of times.  Otherwise, check that the VEA factory is inactive.
 186   void ExpectVEAResponsesForExternalVideoEncoder(
 187       int vea_response_count,
 188       int shm_response_count) const {
 189     if (!vea_factory_)
 190       return;
 191     EXPECT_EQ(vea_response_count, vea_factory_->vea_response_count());
 192     EXPECT_EQ(shm_response_count, vea_factory_->shm_response_count());
 193   }
 194
 195   void SetVEAFactoryAutoRespond(bool auto_respond) {
 196     if (vea_factory_)
 197       vea_factory_->SetAutoRespond(auto_respond);
 198   }
 199
 200  private:
 201   void OnOperationalStatusChange(OperationalStatus status) {
 202     DVLOG(1) << "OnOperationalStatusChange: from " << operational_status_
 203              << " to " << status;
 204     operational_status_ = status;
 205
 206     EXPECT_TRUE(operational_status_ == STATUS_CODEC_REINIT_PENDING ||
 207                 operational_status_ == STATUS_INITIALIZED);
 208
 209     // Create the VideoFrameFactory the first time status changes to
 210     // STATUS_INITIALIZED.
 211     if (operational_status_ == STATUS_INITIALIZED && !video_frame_factory_)
 212       video_frame_factory_ = video_encoder_->CreateVideoFrameFactory().Pass();
 213   }
 214
 215   // Checks that |encoded_frame| matches expected values.  This is the method
 216   // bound in the callback returned from EncodeAndCheckDelivery().
 217   void DeliverEncodedVideoFrame(
 218       uint32 expected_frame_id,
 219       uint32 expected_last_referenced_frame_id,
 220       uint32 expected_rtp_timestamp,
 221       const base::TimeTicks& expected_reference_time,
 222       scoped_ptr<SenderEncodedFrame> encoded_frame) {
 223     EXPECT_TRUE(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 224
 225     EXPECT_EQ(expected_frame_id, encoded_frame->frame_id);
 226     EXPECT_EQ(expected_rtp_timestamp, encoded_frame->rtp_timestamp);
 227     EXPECT_EQ(expected_reference_time, encoded_frame->reference_time);
 228
 229     // The platform encoders are "black boxes" and may choose to vend key frames
 230     // and/or empty data at any time.  The software encoders, however, should
 231     // strictly adhere to expected behavior.
 232     if (is_testing_platform_encoder()) {
 233       const bool expected_key_frame =
 234           expected_frame_id == expected_last_referenced_frame_id;
 235       const bool have_key_frame =
 236           encoded_frame->dependency == EncodedFrame::KEY;
 237       EXPECT_EQ(have_key_frame,
 238                 encoded_frame->frame_id == encoded_frame->referenced_frame_id);
 239       LOG_IF(WARNING, expected_key_frame != have_key_frame)
 240           << "Platform encoder chose to emit a "
 241           << (have_key_frame ? "key" : "delta")
 242           << " frame instead of the expected kind @ frame_id="
 243           << encoded_frame->frame_id;
 244       LOG_IF(WARNING, encoded_frame->data.empty())
 245           << "Platform encoder returned an empty frame @ frame_id="
 246           << encoded_frame->frame_id;
 247     } else {
 248       if (expected_frame_id != expected_last_referenced_frame_id) {
 249         EXPECT_EQ(EncodedFrame::DEPENDENT, encoded_frame->dependency);
 250       } else if (video_config_.max_number_of_video_buffers_used == 1) {
 251         EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency);
 252       }
 253       EXPECT_EQ(expected_last_referenced_frame_id,
 254                 encoded_frame->referenced_frame_id);
 255       EXPECT_FALSE(encoded_frame->data.empty());
 256       ASSERT_TRUE(std::isfinite(encoded_frame->deadline_utilization));
 257       EXPECT_LE(0.0, encoded_frame->deadline_utilization);
 258       ASSERT_TRUE(std::isfinite(encoded_frame->lossy_utilization));
 259       EXPECT_LE(0.0, encoded_frame->lossy_utilization);
 260     }
 261
 262     ++count_frames_delivered_;
 263   }
 264
 265   base::SimpleTestTickClock* const testing_clock_;  // Owned by CastEnvironment.
 266   const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_;
 267   const scoped_refptr<CastEnvironment> cast_environment_;
 268   VideoSenderConfig video_config_;
 269   scoped_ptr<FakeVideoEncodeAcceleratorFactory> vea_factory_;
 270   base::TimeTicks first_frame_time_;
 271   OperationalStatus operational_status_;
 272   scoped_ptr<VideoEncoder> video_encoder_;
 273   scoped_ptr<VideoFrameFactory> video_frame_factory_;
 274
 275   int count_frames_delivered_;
 276
 277   DISALLOW_COPY_AND_ASSIGN(VideoEncoderTest);
 278 };
 279
 280 // A simple test to encode ten frames of video, expecting to see one key frame
 281 // followed by nine delta frames.
 282 TEST_P(VideoEncoderTest, GeneratesKeyFrameThenOnlyDeltaFrames) {
 283   CreateEncoder(false);
 284   SetVEAFactoryAutoRespond(true);
 285
 286   EXPECT_EQ(0, count_frames_delivered());
 287   ExpectVEAResponsesForExternalVideoEncoder(0, 0);
 288
 289   uint32 frame_id = 0;
 290   uint32 reference_frame_id = 0;
 291   const gfx::Size frame_size(1280, 720);
 292
 293   // Some encoders drop one or more frames initially while the encoder
 294   // initializes. Then, for all encoders, expect one key frame is delivered.
 295   bool accepted_first_frame = false;
 296   do {
 297     accepted_first_frame = EncodeAndCheckDelivery(
 298         CreateTestVideoFrame(frame_size), frame_id, reference_frame_id);
 299     if (!encoder_has_resize_delay())
 300       EXPECT_TRUE(accepted_first_frame);
 301     RunTasksAndAdvanceClock();
 302   } while (!accepted_first_frame);
 303   ExpectVEAResponsesForExternalVideoEncoder(1, 3);
 304
 305   // Expect the remaining frames are encoded as delta frames.
 306   for (++frame_id; frame_id < 10; ++frame_id, ++reference_frame_id) {
 307     EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size),
 308                                        frame_id,
 309                                        reference_frame_id));
 310     RunTasksAndAdvanceClock();
 311   }
 312
 313   WaitForAllFramesToBeDelivered(10);
 314   ExpectVEAResponsesForExternalVideoEncoder(1, 3);
 315 }
 316
 317 // Tests basic frame dependency rules when using the VP8 encoder in multi-buffer
 318 // mode.
 319 TEST_P(VideoEncoderTest, FramesDoNotDependOnUnackedFramesInMultiBufferMode) {
 320   if (!is_testing_software_vp8_encoder())
 321     return;  // Only test multibuffer mode for the software VP8 encoder.
 322   CreateEncoder(true);
 323
 324   EXPECT_EQ(0, count_frames_delivered());
 325
 326   const gfx::Size frame_size(1280, 720);
 327   EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 0, 0));
 328   RunTasksAndAdvanceClock();
 329
 330   video_encoder()->LatestFrameIdToReference(0);
 331   EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 1, 0));
 332   RunTasksAndAdvanceClock();
 333
 334   video_encoder()->LatestFrameIdToReference(1);
 335   EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 2, 1));
 336   RunTasksAndAdvanceClock();
 337
 338   video_encoder()->LatestFrameIdToReference(2);
 339
 340   for (uint32 frame_id = 3; frame_id < 10; ++frame_id) {
 341     EXPECT_TRUE(EncodeAndCheckDelivery(
 342         CreateTestVideoFrame(frame_size), frame_id, 2));
 343     RunTasksAndAdvanceClock();
 344   }
 345
 346   EXPECT_EQ(10, count_frames_delivered());
 347 }
 348
 349 // Tests that the encoder continues to output EncodedFrames as the frame size
 350 // changes.  See media/cast/receiver/video_decoder_unittest.cc for a complete
 351 // encode/decode cycle of varied frame sizes that actually checks the frame
 352 // content.
 353 TEST_P(VideoEncoderTest, EncodesVariedFrameSizes) {
 354   CreateEncoder(false);
 355   SetVEAFactoryAutoRespond(true);
 356
 357   EXPECT_EQ(0, count_frames_delivered());
 358   ExpectVEAResponsesForExternalVideoEncoder(0, 0);
 359
 360   std::vector<gfx::Size> frame_sizes;
 361   frame_sizes.push_back(gfx::Size(1280, 720));
 362   frame_sizes.push_back(gfx::Size(640, 360));  // Shrink both dimensions.
 363   frame_sizes.push_back(gfx::Size(300, 200));  // Shrink both dimensions again.
 364   frame_sizes.push_back(gfx::Size(200, 300));  // Same area.
 365   frame_sizes.push_back(gfx::Size(600, 400));  // Grow both dimensions.
 366   frame_sizes.push_back(gfx::Size(638, 400));  // Shrink only one dimension.
 367   frame_sizes.push_back(gfx::Size(638, 398));  // Shrink the other dimension.
 368   frame_sizes.push_back(gfx::Size(320, 180));  // Shrink both dimensions again.
 369   frame_sizes.push_back(gfx::Size(322, 180));  // Grow only one dimension.
 370   frame_sizes.push_back(gfx::Size(322, 182));  // Grow the other dimension.
 371   frame_sizes.push_back(gfx::Size(1920, 1080));  // Grow both dimensions again.
 372
 373   uint32 frame_id = 0;
 374
 375   // Encode one frame at each size. For encoders with a resize delay, except no
 376   // frames to be delivered since each frame size change will sprun
 377   // re-initialization of the underlying encoder. Otherwise expect all key
 378   // frames to come out.
 379   for (const auto& frame_size : frame_sizes) {
 380     EXPECT_EQ(!encoder_has_resize_delay(),
 381               EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), frame_id,
 382                                      frame_id));
 383     RunTasksAndAdvanceClock();
 384     if (!encoder_has_resize_delay())
 385       ++frame_id;
 386   }
 387
 388   // Encode 10+ frames at each size. For encoders with a resize delay, expect
 389   // the first one or more frames are dropped while the encoder re-inits. Then,
 390   // for all encoders, expect one key frame followed by all delta frames.
 391   for (const auto& frame_size : frame_sizes) {
 392     bool accepted_first_frame = false;
 393     do {
 394       accepted_first_frame = EncodeAndCheckDelivery(
 395           CreateTestVideoFrame(frame_size), frame_id, frame_id);
 396       if (!encoder_has_resize_delay())
 397         EXPECT_TRUE(accepted_first_frame);
 398       RunTasksAndAdvanceClock();
 399     } while (!accepted_first_frame);
 400     ++frame_id;
 401     for (int i = 1; i < 10; ++i, ++frame_id) {
 402       EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size),
 403                                          frame_id,
 404                                          frame_id - 1));
 405       RunTasksAndAdvanceClock();
 406     }
 407   }
 408
 409   WaitForAllFramesToBeDelivered(10 * frame_sizes.size());
 410   ExpectVEAResponsesForExternalVideoEncoder(
 411       2 * frame_sizes.size(), 6 * frame_sizes.size());
 412 }
 413
 414 // Verify that everything goes well even if ExternalVideoEncoder is destroyed
 415 // before it has a chance to receive the VEA creation callback.  For all other
 416 // encoders, this tests that the encoder can be safely destroyed before the task
 417 // is run that delivers the first EncodedFrame.
 418 TEST_P(VideoEncoderTest, CanBeDestroyedBeforeVEAIsCreated) {
 419   CreateEncoder(false);
 420
 421   // Send a frame to spawn creation of the ExternalVideoEncoder instance.
 422   EncodeAndCheckDelivery(CreateTestVideoFrame(gfx::Size(1280, 720)), 0, 0);
 423
 424   // Destroy the encoder, and confirm the VEA Factory did not respond yet.
 425   DestroyEncoder();
 426   ExpectVEAResponsesForExternalVideoEncoder(0, 0);
 427
 428   // Allow the VEA Factory to respond by running the creation callback.  When
 429   // the task runs, it will be a no-op since the weak pointers to the
 430   // ExternalVideoEncoder were invalidated.
 431   SetVEAFactoryAutoRespond(true);
 432   RunTasksAndAdvanceClock();
 433   ExpectVEAResponsesForExternalVideoEncoder(1, 0);
 434 }
 435
 436 namespace {
 437 std::vector<std::pair<Codec, bool>> DetermineEncodersToTest() {
 438   std::vector<std::pair<Codec, bool>> values;
 439   // Fake encoder.
 440   values.push_back(std::make_pair(CODEC_VIDEO_FAKE, false));
 441   // Software VP8 encoder.
 442   values.push_back(std::make_pair(CODEC_VIDEO_VP8, false));
 443   // Hardware-accelerated encoder (faked).
 444   values.push_back(std::make_pair(CODEC_VIDEO_VP8, true));
 445 #if defined(OS_MACOSX)
 446   // VideoToolbox encoder (when VideoToolbox is present).
 447   VideoSenderConfig video_config = GetDefaultVideoSenderConfig();
 448   video_config.use_external_encoder = false;
 449   video_config.codec = CODEC_VIDEO_H264;
 450   if (H264VideoToolboxEncoder::IsSupported(video_config))
 451     values.push_back(std::make_pair(CODEC_VIDEO_H264, false));
 452 #endif
 453   return values;
 454 }
 455 }  // namespace
 456
 457 INSTANTIATE_TEST_CASE_P(
 458     , VideoEncoderTest, ::testing::ValuesIn(DetermineEncodersToTest()));
 459
 460 }  // namespace cast
 461 }  // namespace media