remoting/test/test_video_renderer.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/test/test_video_renderer.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/logging.h"
  10 #include "base/synchronization/lock.h"
  11 #include "base/thread_task_runner_handle.h"
  12 #include "base/threading/thread.h"
  13 #include "remoting/codec/video_decoder.h"
  14 #include "remoting/codec/video_decoder_verbatim.h"
  15 #include "remoting/codec/video_decoder_vpx.h"
  16 #include "remoting/proto/video.pb.h"
  17 #include "remoting/test/rgb_value.h"
  18 #include "remoting/test/video_frame_writer.h"
  19 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  20 #include "third_party/webrtc/modules/desktop_capture/shared_desktop_frame.h"
  21
  22 namespace {
  23
  24 // Used to account for frame resizing and lossy encoding error in percentage.
  25 // The average color usually only varies by 1 on each channel, so 0.01 is large
  26 // enough to allow variations while not being flaky for false negative cases.
  27 const double kMaxColorError = 0.01;
  28
  29 }  // namespace
  30
  31 namespace remoting {
  32 namespace test {
  33
  34 // Implements video decoding functionality.
  35 class TestVideoRenderer::Core {
  36  public:
  37   Core();
  38   ~Core();
  39
  40   // Initializes the internal structures of the class.
  41   void Initialize();
  42
  43   // Used to decode video packets.
  44   void ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
  45                           const base::Closure& done);
  46
  47   // Initialize a decoder to decode video packets.
  48   void SetCodecForDecoding(const protocol::ChannelConfig::Codec codec);
  49
  50   // Returns a copy of the current frame.
  51   scoped_ptr<webrtc::DesktopFrame> GetCurrentFrameForTest() const;
  52
  53   // Set expected image pattern for comparison and the callback will be called
  54   // when the pattern is matched.
  55   void ExpectAverageColorInRect(
  56       const webrtc::DesktopRect& expected_rect,
  57       const RGBValue& expected_avg_color,
  58       const base::Closure& image_pattern_matched_callback);
  59
  60   // Turn on/off saving video frames to disk.
  61   void save_frame_data_to_disk(bool save_frame_data_to_disk) {
  62     save_frame_data_to_disk_ = save_frame_data_to_disk;
  63   }
  64
  65  private:
  66   // Returns average color of pixels fall within |rect| on the current frame.
  67   RGBValue CalculateAverageColorValue(const webrtc::DesktopRect& rect) const;
  68
  69   // Compares |candidate_avg_value| to |expected_avg_color_|.
  70   // Returns true if the root mean square of the errors in the R, G and B
  71   // components does not exceed a given limit.
  72   bool ExpectedAverageColorIsMatched(const RGBValue& candidate_avg_value) const;
  73
  74   // Used to ensure Core methods are called on the same thread.
  75   base::ThreadChecker thread_checker_;
  76
  77   // Used to decode video packets.
  78   scoped_ptr<VideoDecoder> decoder_;
  79
  80   // Used to post tasks back to main thread.
  81   scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
  82
  83   // Protects access to |frame_|.
  84   mutable base::Lock lock_;
  85
  86   // Used to store decoded video frame.
  87   scoped_ptr<webrtc::SharedDesktopFrame> frame_;
  88
  89   // Used to store the expected image pattern.
  90   webrtc::DesktopRect expected_rect_;
  91   RGBValue expected_avg_color_;
  92
  93   // Used to store the callback when expected pattern is matched.
  94   base::Closure image_pattern_matched_callback_;
  95
  96   // Used to identify whether saving frame frame data to disk.
  97   bool save_frame_data_to_disk_;
  98
  99   // Used to dump video frames and generate image patterns.
 100   VideoFrameWriter video_frame_writer;
 101
 102   DISALLOW_COPY_AND_ASSIGN(Core);
 103 };
 104
 105 TestVideoRenderer::Core::Core()
 106     : main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
 107       save_frame_data_to_disk_(false) {
 108   thread_checker_.DetachFromThread();
 109 }
 110
 111 TestVideoRenderer::Core::~Core() {
 112   DCHECK(thread_checker_.CalledOnValidThread());
 113 }
 114
 115 void TestVideoRenderer::Core::Initialize() {
 116   DCHECK(thread_checker_.CalledOnValidThread());
 117 }
 118
 119 void TestVideoRenderer::Core::SetCodecForDecoding(
 120     const protocol::ChannelConfig::Codec codec) {
 121   DCHECK(thread_checker_.CalledOnValidThread());
 122
 123   if (decoder_) {
 124     LOG(WARNING) << "Decoder is set more than once";
 125   }
 126
 127   switch (codec) {
 128     case protocol::ChannelConfig::CODEC_VP8: {
 129       VLOG(1) << "Test Video Renderer will use VP8 decoder";
 130       decoder_ = VideoDecoderVpx::CreateForVP8();
 131       break;
 132     }
 133     case protocol::ChannelConfig::CODEC_VP9: {
 134       VLOG(1) << "Test Video Renderer will use VP9 decoder";
 135       decoder_ = VideoDecoderVpx::CreateForVP9();
 136       break;
 137     }
 138     case protocol::ChannelConfig::CODEC_VERBATIM: {
 139       VLOG(1) << "Test Video Renderer will use VERBATIM decoder";
 140       decoder_.reset(new VideoDecoderVerbatim());
 141       break;
 142     }
 143     default: {
 144       NOTREACHED() << "Unsupported codec: " << codec;
 145     }
 146   }
 147 }
 148
 149 scoped_ptr<webrtc::DesktopFrame>
 150 TestVideoRenderer::Core::GetCurrentFrameForTest() const {
 151   base::AutoLock auto_lock(lock_);
 152   DCHECK(frame_);
 153   return make_scoped_ptr(webrtc::BasicDesktopFrame::CopyOf(*frame_));
 154 }
 155
 156 void TestVideoRenderer::Core::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 157                                                  const base::Closure& done) {
 158   DCHECK(thread_checker_.CalledOnValidThread());
 159   DCHECK(decoder_);
 160   DCHECK(packet);
 161
 162   VLOG(2) << "TestVideoRenderer::Core::ProcessVideoPacket() Called";
 163
 164   // Screen size is attached on the first packet as well as when the
 165   // host screen is resized.
 166   if (packet->format().has_screen_width() &&
 167       packet->format().has_screen_height()) {
 168     webrtc::DesktopSize source_size(packet->format().screen_width(),
 169                                     packet->format().screen_height());
 170     if (!frame_ || !frame_->size().equals(source_size)) {
 171       base::AutoLock auto_lock(lock_);
 172       frame_.reset(webrtc::SharedDesktopFrame::Wrap(
 173           new webrtc::BasicDesktopFrame(source_size)));
 174     }
 175   }
 176
 177   // To make life easier, assume that the desktop shape is a single rectangle.
 178   packet->clear_use_desktop_shape();
 179
 180   // Render the result into a new DesktopFrame instance that shares buffer with
 181   // |frame_|. updated_region() will be updated for |new_frame|, but not for
 182   // |frame_|.
 183   scoped_ptr<webrtc::DesktopFrame> new_frame(frame_->Share());
 184
 185   {
 186     base::AutoLock auto_lock(lock_);
 187     if (!decoder_->DecodePacket(*packet, new_frame.get())) {
 188       LOG(ERROR) << "Decoder::DecodePacket() failed.";
 189       return;
 190     }
 191   }
 192
 193   main_task_runner_->PostTask(FROM_HERE, done);
 194
 195   if (save_frame_data_to_disk_) {
 196     scoped_ptr<webrtc::DesktopFrame> frame(
 197         webrtc::BasicDesktopFrame::CopyOf(*frame_));
 198     video_frame_writer.HighlightRectInFrame(frame.get(), expected_rect_);
 199     video_frame_writer.WriteFrameToDefaultPath(*frame);
 200   }
 201
 202   // Check to see if a image pattern matched reply is passed in, and whether
 203   // the |expected_rect_| falls within the current frame.
 204   if (image_pattern_matched_callback_.is_null() ||
 205       expected_rect_.right() > frame_->size().width() ||
 206       expected_rect_.bottom() > frame_->size().height()) {
 207     return;
 208   }
 209   // Compare the expected image pattern with the corresponding rectangle
 210   // region
 211   // on the current frame.
 212   RGBValue accumulating_avg_value = CalculateAverageColorValue(expected_rect_);
 213   if (ExpectedAverageColorIsMatched(accumulating_avg_value)) {
 214     main_task_runner_->PostTask(
 215         FROM_HERE, base::ResetAndReturn(&image_pattern_matched_callback_));
 216   }
 217 }
 218
 219 void TestVideoRenderer::Core::ExpectAverageColorInRect(
 220     const webrtc::DesktopRect& expected_rect,
 221     const RGBValue& expected_avg_color,
 222     const base::Closure& image_pattern_matched_callback) {
 223   DCHECK(thread_checker_.CalledOnValidThread());
 224
 225   expected_rect_ = expected_rect;
 226   expected_avg_color_ = expected_avg_color;
 227   image_pattern_matched_callback_ = image_pattern_matched_callback;
 228 }
 229
 230 RGBValue TestVideoRenderer::Core::CalculateAverageColorValue(
 231     const webrtc::DesktopRect& rect) const {
 232   int red_sum = 0;
 233   int green_sum = 0;
 234   int blue_sum = 0;
 235
 236   // Loop through pixels that fall within |accumulating_rect_| to obtain the
 237   // average color value.
 238   for (int y = rect.top(); y < rect.bottom(); ++y) {
 239     uint8_t* frame_pos =
 240         frame_->data() + (y * frame_->stride() +
 241                           rect.left() * webrtc::DesktopFrame::kBytesPerPixel);
 242
 243     // Pixels of decoded video frame are presented in ARGB format.
 244     for (int x = 0; x < rect.width(); ++x) {
 245       red_sum += frame_pos[2];
 246       green_sum += frame_pos[1];
 247       blue_sum += frame_pos[0];
 248       frame_pos += 4;
 249     }
 250   }
 251
 252   int area = rect.width() * rect.height();
 253   RGBValue rgb_value(red_sum / area, green_sum / area, blue_sum / area);
 254   return rgb_value;
 255 }
 256
 257 bool TestVideoRenderer::Core::ExpectedAverageColorIsMatched(
 258     const RGBValue& candidate_avg_value) const {
 259   double error_sum_squares = 0;
 260   double red_error = expected_avg_color_.red - candidate_avg_value.red;
 261   double green_error = expected_avg_color_.green - candidate_avg_value.green;
 262   double blue_error = expected_avg_color_.blue - candidate_avg_value.blue;
 263   error_sum_squares = red_error * red_error + green_error * green_error +
 264                       blue_error * blue_error;
 265   error_sum_squares /= (255.0 * 255.0);
 266
 267   return sqrt(error_sum_squares / 3) < kMaxColorError;
 268 }
 269
 270 TestVideoRenderer::TestVideoRenderer()
 271     : video_decode_thread_(
 272         new base::Thread("TestVideoRendererVideoDecodingThread")),
 273       weak_factory_(this) {
 274   DCHECK(thread_checker_.CalledOnValidThread());
 275
 276   core_.reset(new Core());
 277   if (!video_decode_thread_->Start()) {
 278     LOG(ERROR) << "Cannot start TestVideoRenderer";
 279   } else {
 280     video_decode_task_runner_ = video_decode_thread_->task_runner();
 281     video_decode_task_runner_->PostTask(FROM_HERE, base::Bind(&Core::Initialize,
 282                                         base::Unretained(core_.get())));
 283   }
 284 }
 285
 286 TestVideoRenderer::~TestVideoRenderer() {
 287   DCHECK(thread_checker_.CalledOnValidThread());
 288
 289   video_decode_task_runner_->DeleteSoon(FROM_HERE, core_.release());
 290
 291   // The thread's message loop will run until it runs out of work.
 292   video_decode_thread_->Stop();
 293 }
 294
 295 void TestVideoRenderer::OnSessionConfig(const protocol::SessionConfig& config) {
 296   DCHECK(thread_checker_.CalledOnValidThread());
 297
 298   VLOG(2) << "TestVideoRenderer::OnSessionConfig() Called";
 299   protocol::ChannelConfig::Codec codec = config.video_config().codec;
 300   SetCodecForDecoding(codec);
 301 }
 302
 303 ChromotingStats* TestVideoRenderer::GetStats() {
 304   DCHECK(thread_checker_.CalledOnValidThread());
 305
 306   VLOG(2) << "TestVideoRenderer::GetStats() Called";
 307   return nullptr;
 308 }
 309
 310 protocol::VideoStub* TestVideoRenderer::GetVideoStub() {
 311   DCHECK(thread_checker_.CalledOnValidThread());
 312
 313   VLOG(2) << "TestVideoRenderer::GetVideoStub() Called";
 314   return this;
 315 }
 316
 317 void TestVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> video_packet,
 318                                            const base::Closure& done) {
 319   DCHECK(thread_checker_.CalledOnValidThread());
 320   DCHECK(video_decode_task_runner_) << "Failed to start video decode thread";
 321
 322   if (video_packet->has_data() && video_packet->data().size() != 0) {
 323     VLOG(2) << "process video packet is called!";
 324
 325     // Post video process task to the video decode thread.
 326     base::Closure process_video_task = base::Bind(
 327         &TestVideoRenderer::Core::ProcessVideoPacket,
 328         base::Unretained(core_.get()), base::Passed(&video_packet), done);
 329     video_decode_task_runner_->PostTask(FROM_HERE, process_video_task);
 330   } else {
 331     // Log at a high verbosity level as we receive empty packets frequently and
 332     // they can clutter up the debug output if the level is set too low.
 333     VLOG(3) << "Empty Video Packet received.";
 334     done.Run();
 335   }
 336 }
 337
 338 void TestVideoRenderer::SetCodecForDecoding(
 339     const protocol::ChannelConfig::Codec codec) {
 340   DCHECK(thread_checker_.CalledOnValidThread());
 341
 342   VLOG(2) << "TestVideoRenderer::SetDecoder() Called";
 343   video_decode_task_runner_->PostTask(
 344       FROM_HERE, base::Bind(&Core::SetCodecForDecoding,
 345                             base::Unretained(core_.get()),
 346                             codec));
 347 }
 348
 349 scoped_ptr<webrtc::DesktopFrame> TestVideoRenderer::GetCurrentFrameForTest()
 350     const {
 351   DCHECK(thread_checker_.CalledOnValidThread());
 352
 353   return core_->GetCurrentFrameForTest();
 354 }
 355
 356 void TestVideoRenderer::ExpectAverageColorInRect(
 357     const webrtc::DesktopRect& expected_rect,
 358     const RGBValue& expected_avg_color,
 359     const base::Closure& image_pattern_matched_callback) {
 360   DCHECK(thread_checker_.CalledOnValidThread());
 361   DCHECK(!expected_rect.is_empty()) << "Expected rect cannot be empty";
 362
 363   DVLOG(2) << "TestVideoRenderer::SetImagePatternAndMatchedCallback() Called";
 364   video_decode_task_runner_->PostTask(
 365       FROM_HERE,
 366       base::Bind(&Core::ExpectAverageColorInRect, base::Unretained(core_.get()),
 367                  expected_rect, expected_avg_color,
 368                  image_pattern_matched_callback));
 369 }
 370
 371 void TestVideoRenderer::SaveFrameDataToDisk(bool save_frame_data_to_disk) {
 372   DCHECK(thread_checker_.CalledOnValidThread());
 373
 374   video_decode_task_runner_->PostTask(
 375       FROM_HERE,
 376       base::Bind(&Core::save_frame_data_to_disk, base::Unretained(core_.get()),
 377                  save_frame_data_to_disk));
 378 }
 379
 380 }  // namespace test
 381 }  // namespace remoting