From 497e9a05decbcd344d774e52c1b89cdece525d99 Mon Sep 17 00:00:00 2001 From: liaoyuke Date: Fri, 26 Jun 2015 12:47:35 -0700 Subject: [PATCH] Updated Test Driver with video decoding capabilities so that when a video sample is received from remote host, it can be decoded and rendered to buffer in memory. BUG= Review URL: https://codereview.chromium.org/1190383007 Cr-Commit-Position: refs/heads/master@{#336445} --- remoting/BUILD.gn | 1 + remoting/remoting_test.gypi | 1 + remoting/test/DEPS | 1 + remoting/test/test_video_renderer.cc | 174 +++++++++++++++++- remoting/test/test_video_renderer.h | 32 +++- remoting/test/test_video_renderer_unittest.cc | 247 ++++++++++++++++++++++++++ 6 files changed, 446 insertions(+), 10 deletions(-) create mode 100644 remoting/test/test_video_renderer_unittest.cc diff --git a/remoting/BUILD.gn b/remoting/BUILD.gn index 899f9d8c416c..314a484f70ff 100644 --- a/remoting/BUILD.gn +++ b/remoting/BUILD.gn @@ -150,6 +150,7 @@ if (!is_mac) { "test/app_remoting_test_driver_environment_unittest.cc", "test/remote_host_info_fetcher_unittest.cc", "test/test_chromoting_client_unittest.cc", + "test/test_video_renderer_unittest.cc", ] configs += [ diff --git a/remoting/remoting_test.gypi b/remoting/remoting_test.gypi index 807e86e5a6ba..3d284f7c5c7a 100644 --- a/remoting/remoting_test.gypi +++ b/remoting/remoting_test.gypi @@ -357,6 +357,7 @@ 'test/app_remoting_test_driver_environment_unittest.cc', 'test/remote_host_info_fetcher_unittest.cc', 'test/test_chromoting_client_unittest.cc', + 'test/test_video_renderer_unittest.cc', ], 'conditions': [ [ 'OS=="win"', { diff --git a/remoting/test/DEPS b/remoting/test/DEPS index 58a26bab3f2f..f5971fe461be 100644 --- a/remoting/test/DEPS +++ b/remoting/test/DEPS @@ -2,6 +2,7 @@ include_rules = [ "+jingle/glue", "+net", "+remoting/client", + "+remoting/codec", "+remoting/host", "+remoting/protocol", "+remoting/signaling", diff --git a/remoting/test/test_video_renderer.cc b/remoting/test/test_video_renderer.cc index 864356a112cb..9a9e66fed988 100644 --- a/remoting/test/test_video_renderer.cc +++ b/remoting/test/test_video_renderer.cc @@ -4,20 +4,173 @@ #include "remoting/test/test_video_renderer.h" +#include "base/bind.h" #include "base/logging.h" +#include "base/synchronization/lock.h" +#include "base/threading/thread.h" +#include "remoting/codec/video_decoder.h" +#include "remoting/codec/video_decoder_verbatim.h" +#include "remoting/codec/video_decoder_vpx.h" #include "remoting/proto/video.pb.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_region.h" namespace remoting { namespace test { -TestVideoRenderer::TestVideoRenderer() : video_frames_processed_(0) { +// Implements video decoding functionality. +class TestVideoRenderer::Core { + public: + Core(); + ~Core(); + + // Initializes the internal structures of the class. + void Initialize(); + + // Set negotiated session configuration. + void OnSessionConfig(const protocol::SessionConfig& config); + + // Used to decode video packets. + void ProcessVideoPacket(scoped_ptr packet); + + // Returns a copy of the current buffer. + scoped_ptr GetBufferForTest() const; + + private: + // Used to ensure TestVideoRenderer::Core methods are called on the same + // thread. + base::ThreadChecker thread_checker_; + + // Used to decode video packets. + scoped_ptr decoder_; + + // Updated region of the current desktop frame compared to previous one. + webrtc::DesktopRegion updated_region_; + + // Screen size of the remote host. + webrtc::DesktopSize screen_size_; + + // Used to store decoded video frame. + scoped_ptr buffer_; + + // Protects access to |buffer_|. + mutable base::Lock lock_; + + DISALLOW_COPY_AND_ASSIGN(Core); +}; + +TestVideoRenderer::Core::Core() { + thread_checker_.DetachFromThread(); +} + +TestVideoRenderer::Core::~Core() { + DCHECK(thread_checker_.CalledOnValidThread()); +} + +void TestVideoRenderer::Core::Initialize() { + DCHECK(thread_checker_.CalledOnValidThread()); +} + +void TestVideoRenderer::Core::OnSessionConfig( + const protocol::SessionConfig& config) { + DCHECK(thread_checker_.CalledOnValidThread()); + + protocol::ChannelConfig::Codec codec = config.video_config().codec; + switch(codec) { + case protocol::ChannelConfig::CODEC_VP8: { + DVLOG(2) << "Test Video Renderer will use VP8 decoder"; + decoder_ = VideoDecoderVpx::CreateForVP8(); + break; + } + case protocol::ChannelConfig::CODEC_VP9: { + DVLOG(2) << "Test Video Renderer will use VP9 decoder"; + decoder_ = VideoDecoderVpx::CreateForVP9(); + break; + } + case protocol::ChannelConfig::CODEC_VERBATIM: { + DVLOG(2) << "Test Video Renderer will use VERBATIM decoder"; + decoder_.reset(new VideoDecoderVerbatim()); + break; + } + default: { + NOTREACHED() << "Unsupported codec: " << codec; + } + } +} + +scoped_ptr + TestVideoRenderer::Core::GetBufferForTest() const { + base::AutoLock auto_lock(lock_); + return make_scoped_ptr(webrtc::BasicDesktopFrame::CopyOf(*buffer_.get())); +} + +void TestVideoRenderer::Core::ProcessVideoPacket( + scoped_ptr packet) { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(decoder_); + DCHECK(packet); + + DVLOG(2) << "TestVideoRenderer::Core::ProcessVideoPacket() Called"; + + // Screen size is attached on the first packet as well as when the + // host screen is resized. + if (packet->format().has_screen_width() && + packet->format().has_screen_height()) { + webrtc::DesktopSize source_size(packet->format().screen_width(), + packet->format().screen_height()); + if (!screen_size_.equals(source_size)) { + screen_size_ = source_size; + decoder_->Initialize(screen_size_); + buffer_.reset(new webrtc::BasicDesktopFrame(screen_size_)); + } + } + + // To make life easier, assume that the desktop shape is a single rectangle. + packet->clear_use_desktop_shape(); + if (!decoder_->DecodePacket(*packet.get())) { + LOG(ERROR) << "Decoder::DecodePacket() failed."; + return; + } + { + base::AutoLock auto_lock(lock_); + + // Render the decoded packet and write results to the buffer. + // Note that the |updated_region_| maintains the changed regions compared to + // previous video frame. + decoder_->RenderFrame(screen_size_, + webrtc::DesktopRect::MakeWH(screen_size_.width(), + screen_size_.height()), buffer_->data(), + buffer_->stride(), &updated_region_); + } +} + +TestVideoRenderer::TestVideoRenderer() + : core_(new Core()), + video_decode_thread_( + new base::Thread("TestVideoRendererVideoDecodingThread")) { + if (!video_decode_thread_->Start()) { + LOG(ERROR) << "Cannot start TestVideoRenderer"; + } else { + video_decode_task_runner_ = video_decode_thread_->task_runner(); + video_decode_task_runner_->PostTask(FROM_HERE, base::Bind(&Core::Initialize, + base::Unretained(core_.get()))); + } } TestVideoRenderer::~TestVideoRenderer() { + DCHECK(thread_checker_.CalledOnValidThread()); + + video_decode_task_runner_->DeleteSoon(FROM_HERE, core_.release()); + + // The thread's message loop will run until it runs out of work. + video_decode_thread_->Stop(); } void TestVideoRenderer::OnSessionConfig(const protocol::SessionConfig& config) { DVLOG(2) << "TestVideoRenderer::OnSessionConfig() Called"; + video_decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&Core::OnSessionConfig, + base::Unretained(core_.get()), config)); } ChromotingStats* TestVideoRenderer::GetStats() { @@ -32,18 +185,27 @@ protocol::VideoStub* TestVideoRenderer::GetVideoStub() { void TestVideoRenderer::ProcessVideoPacket(scoped_ptr video_packet, const base::Closure& done) { - if (!video_packet->data().empty()) { - // If the video frame was not a keep alive frame (i.e. not empty) then - // count it. - DVLOG(2) << "Video Packet Processed, Total: " << ++video_frames_processed_; + DCHECK(video_decode_task_runner_) << "Failed to start video decode thread"; + + if (video_packet->has_data() && video_packet->data().size() != 0) { + DVLOG(2) << "process video packet is called!"; + + // Post video process task to the video decode thread. + base::Closure process_video_task = base::Bind( + &TestVideoRenderer::Core::ProcessVideoPacket, + base::Unretained(core_.get()), base::Passed(&video_packet)); + video_decode_task_runner_->PostTask(FROM_HERE, process_video_task); } else { // Log at a high verbosity level as we receive empty packets frequently and // they can clutter up the debug output if the level is set too low. DVLOG(3) << "Empty Video Packet received."; } - done.Run(); } +scoped_ptr TestVideoRenderer::GetBufferForTest() const { + return core_->GetBufferForTest(); +} + } // namespace test } // namespace remoting diff --git a/remoting/test/test_video_renderer.h b/remoting/test/test_video_renderer.h index fcf122abcd9a..acaa7397a7c8 100644 --- a/remoting/test/test_video_renderer.h +++ b/remoting/test/test_video_renderer.h @@ -5,11 +5,22 @@ #ifndef REMOTING_TEST_TEST_VIDEO_RENDERER_H_ #define REMOTING_TEST_TEST_VIDEO_RENDERER_H_ -#include "base/callback.h" #include "base/memory/scoped_ptr.h" +#include "base/threading/thread_checker.h" +#include "media/base/video_frame.h" #include "remoting/client/video_renderer.h" +#include "remoting/protocol/session_config.h" #include "remoting/protocol/video_stub.h" +namespace base { +class Thread; +class SingleThreadTaskRunner; +} + +namespace webrtc { +class DesktopFrame; +} + namespace remoting { namespace test { @@ -24,13 +35,26 @@ class TestVideoRenderer : public VideoRenderer, public protocol::VideoStub { ChromotingStats* GetStats() override; protocol::VideoStub* GetVideoStub() override; - private: // protocol::VideoStub interface. void ProcessVideoPacket(scoped_ptr video_packet, const base::Closure& done) override; - // Track the number of populated video frames which have been received. - int video_frames_processed_; + // Returns a copy of the current buffer. + scoped_ptr GetBufferForTest() const; + + private: + // The actual implementation resides in Core class. + class Core; + scoped_ptr core_; + + // Used to ensure TestVideoRenderer methods are called on the same thread. + base::ThreadChecker thread_checker_; + + // Used to decode and process video packets. + scoped_ptr video_decode_thread_; + + // Used to post tasks to video decode thread. + scoped_refptr video_decode_task_runner_; DISALLOW_COPY_AND_ASSIGN(TestVideoRenderer); }; diff --git a/remoting/test/test_video_renderer_unittest.cc b/remoting/test/test_video_renderer_unittest.cc new file mode 100644 index 000000000000..7ee9b028797b --- /dev/null +++ b/remoting/test/test_video_renderer_unittest.cc @@ -0,0 +1,247 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "remoting/test/test_video_renderer.h" + +#include + +#include "base/memory/scoped_vector.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/timer/timer.h" +#include "media/base/video_frame.h" +#include "remoting/codec/video_encoder.h" +#include "remoting/codec/video_encoder_verbatim.h" +#include "remoting/codec/video_encoder_vpx.h" +#include "remoting/proto/video.pb.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_region.h" + +namespace { +const int kBytesPerPixel = 4; +const int kDefaultScreenWidth = 1024; +const int kDefaultScreenHeight = 768; +const double kDefaultErrorLimit = 0.02; +} + +namespace remoting { +namespace test { + +// Provides basic functionality for for the TestVideoRenderer Tests below. +// This fixture also creates an MessageLoop to test decoding video packets. +class TestVideoRendererTest : public testing::Test { + public: + TestVideoRendererTest(); + ~TestVideoRendererTest() override; + + // Generate a frame containing a gradient and test decoding of + // TestVideoRenderer. The original frame is compared to the one obtained from + // decoding the video packet, and the error at each pixel is the root mean + // square of the errors in the R, G and B components, each normalized to + // [0, 1]. This routine checks that the mean error over all pixels do not + // exceed a given limit. + void TestVideoPacketProcessing(int screen_width, int screen_height, + double error_limit) ; + + // Generate a basic desktop frame containing a gradient. + scoped_ptr CreateDesktopFrameWithGradient( + int screen_width, int screen_height) const; + + protected: + // Used to post tasks to the message loop. + scoped_ptr run_loop_; + + // Used to set timeouts and delays. + scoped_ptr timer_; + + // Manages the decoder and process generated video packets. + scoped_ptr test_video_renderer_; + + // Used to encode desktop frames to generate video packets. + scoped_ptr encoder_; + + private: + // testing::Test interface. + void SetUp() override; + + // return the mean error of two frames. + double CalculateError(const webrtc::DesktopFrame* original_frame, + const webrtc::DesktopFrame* decoded_frame) const; + + // Fill a desktop frame with a gradient. + void FillFrameWithGradient(webrtc::DesktopFrame* frame) const; + + // The thread's message loop. Valid only when the thread is alive. + scoped_ptr message_loop_; + + DISALLOW_COPY_AND_ASSIGN(TestVideoRendererTest); +}; + +TestVideoRendererTest::TestVideoRendererTest() + : timer_(new base::Timer(true, false)) {} + +TestVideoRendererTest::~TestVideoRendererTest() {} + +void TestVideoRendererTest::SetUp() { + if (!base::MessageLoop::current()) { + // Create a temporary message loop if the current thread does not already + // have one. + message_loop_.reset(new base::MessageLoop); + } + test_video_renderer_.reset(new TestVideoRenderer()); +} + +void TestVideoRendererTest::TestVideoPacketProcessing(int screen_width, + int screen_height, + double error_limit) { + scoped_ptr original_frame = + CreateDesktopFrameWithGradient(screen_width, screen_height); + EXPECT_TRUE(original_frame.get()); + + scoped_ptr packet = encoder_->Encode(*original_frame.get()); + test_video_renderer_->ProcessVideoPacket(packet.Pass(), + base::Bind(&base::DoNothing)); + + // wait for long enough so that the video packet is decoded and rendered to + // the buffer. + run_loop_.reset(new base::RunLoop()); + timer_->Start(FROM_HERE, base::TimeDelta::FromMilliseconds(100), + run_loop_->QuitClosure()); + run_loop_->Run(); + timer_->Stop(); + + scoped_ptr buffer_copy = + test_video_renderer_->GetBufferForTest(); + EXPECT_NE(buffer_copy, nullptr); + double error = CalculateError(original_frame.get(), buffer_copy.get()); + EXPECT_LT(error, error_limit); +} + +double TestVideoRendererTest::CalculateError( + const webrtc::DesktopFrame* original_frame, + const webrtc::DesktopFrame* decoded_frame) const { + // Check size remains the same after encoding and decoding. + EXPECT_EQ(original_frame->size().width(), decoded_frame->size().width()); + EXPECT_EQ(original_frame->size().height(), decoded_frame->size().height()); + EXPECT_EQ(original_frame->stride(), decoded_frame->stride()); + int screen_width = original_frame->size().width(); + int screen_height = original_frame->size().height(); + + // Error is calculated as the sum of the square error at each pixel in the + // R, G and B components, each normalized to [0, 1] + double error_sum_squares = 0.0; + + // The mapping between the position of a pixel on 3-dimensional image + // (origin at top left corner) and its position in 1-dimensional buffer. + // + // _______________ + // | | | stride = 4 * width; + // | | | + // | | height | height * stride + width + 0; Red channel. + // | | | => height * stride + width + 1; Green channel. + // |------- | height * stride + width + 2; Blue channel. + // | width | + // |_______________| + // + for (int height = 0; height < screen_height; ++height) { + uint8_t* original_ptr = original_frame->data() + + height * original_frame->stride(); + uint8_t* decoded_ptr = decoded_frame->data() + + height * decoded_frame->stride(); + + for (int width = 0; width < screen_width; ++width) { + // Errors are calculated in the R, G, B components. + for (int j = 0; j < 3; ++j) { + int offset = kBytesPerPixel * width + j; + double original_value = static_cast(*(original_ptr + offset)); + double decoded_value = static_cast(*(decoded_ptr + offset)); + double error = original_value - decoded_value; + + // Normalize the error to [0, 1]. + error /= 255.0; + error_sum_squares += error * error; + } + } + } + return sqrt(error_sum_squares / (3 * screen_width * screen_height)); +} + +scoped_ptr + TestVideoRendererTest::CreateDesktopFrameWithGradient( + int screen_width, int screen_height) const { + webrtc::DesktopSize screen_size(screen_width, screen_height); + scoped_ptr frame( + new webrtc::BasicDesktopFrame(screen_size)); + frame->mutable_updated_region()->SetRect( + webrtc::DesktopRect::MakeSize(screen_size)); + FillFrameWithGradient(frame.get()); + return frame.Pass(); +} + +void TestVideoRendererTest::FillFrameWithGradient( + webrtc::DesktopFrame* frame) const { + for (int y = 0; y < frame->size().height(); ++y) { + uint8* p = frame->data() + y * frame->stride(); + for (int x = 0; x < frame->size().width(); ++x) { + *p++ = (255.0 * x) / frame->size().width(); + *p++ = (164.0 * y) / frame->size().height(); + *p++ = (82.0 * (x + y)) / + (frame->size().width() + frame->size().height()); + *p++ = 0; + } + } +} + +// Verify video packets are processed correctly. +TEST_F(TestVideoRendererTest, VerifyVideoProcessing) { + encoder_ = VideoEncoderVpx::CreateForVP8(); + scoped_ptr config = + protocol::SessionConfig::ForTest(); + test_video_renderer_->OnSessionConfig(*config.get()); + + TestVideoPacketProcessing(kDefaultScreenWidth, kDefaultScreenHeight, + kDefaultErrorLimit); + + // Post multiple tasks to |test_video_renderer_|, and it should not crash. + // 20 is chosen because it's large enough to make sure that there will be + // more than one task on the video decode thread, while not too large to wait + // for too long for the unit test to complete. + int task_num = 20; + ScopedVector video_packets; + for (int i = 0; i < task_num; ++i){ + scoped_ptr original_frame = + CreateDesktopFrameWithGradient(kDefaultScreenWidth, + kDefaultScreenHeight); + video_packets.push_back(encoder_->Encode(*original_frame.get())); + } + + for (int i = 0; i < task_num; ++i) { + // Transfer ownership of video packet. + VideoPacket* packet = video_packets[i]; + video_packets[i] = nullptr; + test_video_renderer_->ProcessVideoPacket(make_scoped_ptr(packet), + base::Bind(&base::DoNothing)); + } +} + +// Verify video packet size change is handled properly. +TEST_F(TestVideoRendererTest, VerifyVideoPacketSizeChange) { + encoder_ = VideoEncoderVpx::CreateForVP8(); + scoped_ptr config = + protocol::SessionConfig::ForTest(); + test_video_renderer_->OnSessionConfig(*config.get()); + + TestVideoPacketProcessing(kDefaultScreenWidth, kDefaultScreenHeight, + kDefaultErrorLimit); + + TestVideoPacketProcessing(2 * kDefaultScreenWidth, 2 * kDefaultScreenHeight, + kDefaultErrorLimit); + + TestVideoPacketProcessing(kDefaultScreenWidth / 2, kDefaultScreenHeight / 2, + kDefaultErrorLimit); +} + +} // namespace test +} // namespace remoting -- 2.11.4.GIT