remoting/client/software_video_renderer.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/software_video_renderer.h"
   6
   7 #include <list>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/callback_helpers.h"
  12 #include "base/location.h"
  13 #include "base/logging.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "remoting/base/util.h"
  16 #include "remoting/client/frame_consumer.h"
  17 #include "remoting/codec/video_decoder.h"
  18 #include "remoting/codec/video_decoder_verbatim.h"
  19 #include "remoting/codec/video_decoder_vpx.h"
  20 #include "remoting/protocol/session_config.h"
  21 #include "third_party/libyuv/include/libyuv/convert_argb.h"
  22 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  23
  24 using base::Passed;
  25 using remoting::protocol::ChannelConfig;
  26 using remoting::protocol::SessionConfig;
  27
  28 namespace remoting {
  29
  30 // This class wraps a VideoDecoder and byte-swaps the pixels for compatibility
  31 // with the android.graphics.Bitmap class.
  32 // TODO(lambroslambrou): Refactor so that the VideoDecoder produces data
  33 // in the right byte-order, instead of swapping it here.
  34 class RgbToBgrVideoDecoderFilter : public VideoDecoder {
  35  public:
  36   RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent)
  37       : parent_(parent.Pass()) {
  38   }
  39
  40   void Initialize(const webrtc::DesktopSize& screen_size) override {
  41     parent_->Initialize(screen_size);
  42   }
  43
  44   bool DecodePacket(const VideoPacket& packet) override {
  45     return parent_->DecodePacket(packet);
  46   }
  47
  48   void Invalidate(const webrtc::DesktopSize& view_size,
  49                   const webrtc::DesktopRegion& region) override {
  50     return parent_->Invalidate(view_size, region);
  51   }
  52
  53   void RenderFrame(const webrtc::DesktopSize& view_size,
  54                    const webrtc::DesktopRect& clip_area,
  55                    uint8* image_buffer,
  56                    int image_stride,
  57                    webrtc::DesktopRegion* output_region) override {
  58     parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride,
  59                          output_region);
  60
  61     for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd();
  62          i.Advance()) {
  63       webrtc::DesktopRect rect = i.rect();
  64       uint8* pixels = image_buffer + (rect.top() * image_stride) +
  65         (rect.left() * kBytesPerPixel);
  66       libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride,
  67                          rect.width(), rect.height());
  68     }
  69   }
  70
  71   const webrtc::DesktopRegion* GetImageShape() override {
  72     return parent_->GetImageShape();
  73   }
  74
  75  private:
  76   scoped_ptr<VideoDecoder> parent_;
  77 };
  78
  79 class SoftwareVideoRenderer::Core {
  80  public:
  81   Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
  82        scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
  83        scoped_ptr<FrameConsumerProxy> consumer);
  84   ~Core();
  85
  86   void OnSessionConfig(const protocol::SessionConfig& config);
  87   void DrawBuffer(webrtc::DesktopFrame* buffer);
  88   void InvalidateRegion(const webrtc::DesktopRegion& region);
  89   void RequestReturnBuffers(const base::Closure& done);
  90   void SetOutputSizeAndClip(
  91       const webrtc::DesktopSize& view_size,
  92       const webrtc::DesktopRect& clip_area);
  93
  94   // Decodes the contents of |packet|. DecodePacket may keep a reference to
  95   // |packet| so the |packet| must remain alive and valid until |done| is
  96   // executed.
  97   void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done);
  98
  99  private:
 100   // Paints the invalidated region to the next available buffer and returns it
 101   // to the consumer.
 102   void SchedulePaint();
 103   void DoPaint();
 104
 105   scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
 106   scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_;
 107   scoped_ptr<FrameConsumerProxy> consumer_;
 108   scoped_ptr<VideoDecoder> decoder_;
 109
 110   // Remote screen size in pixels.
 111   webrtc::DesktopSize source_size_;
 112
 113   // Vertical and horizontal DPI of the remote screen.
 114   webrtc::DesktopVector source_dpi_;
 115
 116   // The current dimensions of the frame consumer view.
 117   webrtc::DesktopSize view_size_;
 118   webrtc::DesktopRect clip_area_;
 119
 120   // The drawing buffers supplied by the frame consumer.
 121   std::list<webrtc::DesktopFrame*> buffers_;
 122
 123   // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint().
 124   bool paint_scheduled_;
 125
 126   base::WeakPtrFactory<Core> weak_factory_;
 127 };
 128
 129 SoftwareVideoRenderer::Core::Core(
 130     scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
 131     scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
 132     scoped_ptr<FrameConsumerProxy> consumer)
 133     : main_task_runner_(main_task_runner),
 134       decode_task_runner_(decode_task_runner),
 135       consumer_(consumer.Pass()),
 136       paint_scheduled_(false),
 137       weak_factory_(this) {}
 138
 139 SoftwareVideoRenderer::Core::~Core() {
 140 }
 141
 142 void SoftwareVideoRenderer::Core::OnSessionConfig(const SessionConfig& config) {
 143   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 144
 145   // Initialize decoder based on the selected codec.
 146   ChannelConfig::Codec codec = config.video_config().codec;
 147   if (codec == ChannelConfig::CODEC_VERBATIM) {
 148     decoder_.reset(new VideoDecoderVerbatim());
 149   } else if (codec == ChannelConfig::CODEC_VP8) {
 150     decoder_ = VideoDecoderVpx::CreateForVP8();
 151   } else if (codec == ChannelConfig::CODEC_VP9) {
 152     decoder_ = VideoDecoderVpx::CreateForVP9();
 153   } else {
 154     NOTREACHED() << "Invalid Encoding found: " << codec;
 155   }
 156
 157   if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) {
 158     scoped_ptr<VideoDecoder> wrapper(
 159         new RgbToBgrVideoDecoderFilter(decoder_.Pass()));
 160     decoder_ = wrapper.Pass();
 161   }
 162 }
 163
 164 void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet,
 165                                                 const base::Closure& done) {
 166   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 167
 168   bool decoder_needs_reset = false;
 169   bool notify_size_or_dpi_change = false;
 170
 171   // If the packet includes screen size or DPI information, store them.
 172   if (packet->format().has_screen_width() &&
 173       packet->format().has_screen_height()) {
 174     webrtc::DesktopSize source_size(packet->format().screen_width(),
 175                                     packet->format().screen_height());
 176     if (!source_size_.equals(source_size)) {
 177       source_size_ = source_size;
 178       decoder_needs_reset = true;
 179       notify_size_or_dpi_change = true;
 180     }
 181   }
 182   if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
 183     webrtc::DesktopVector source_dpi(packet->format().x_dpi(),
 184                                      packet->format().y_dpi());
 185     if (!source_dpi.equals(source_dpi_)) {
 186       source_dpi_ = source_dpi;
 187       notify_size_or_dpi_change = true;
 188     }
 189   }
 190
 191   // If we've never seen a screen size, ignore the packet.
 192   if (source_size_.is_empty()) {
 193     main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
 194     return;
 195   }
 196
 197   if (decoder_needs_reset)
 198     decoder_->Initialize(source_size_);
 199   if (notify_size_or_dpi_change)
 200     consumer_->SetSourceSize(source_size_, source_dpi_);
 201
 202   if (decoder_->DecodePacket(*packet.get())) {
 203     SchedulePaint();
 204   } else {
 205     LOG(ERROR) << "DecodePacket() failed.";
 206   }
 207
 208   main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
 209 }
 210
 211 void SoftwareVideoRenderer::Core::SchedulePaint() {
 212   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 213   if (paint_scheduled_)
 214     return;
 215   paint_scheduled_ = true;
 216   decode_task_runner_->PostTask(
 217       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint,
 218                             weak_factory_.GetWeakPtr()));
 219 }
 220
 221 void SoftwareVideoRenderer::Core::DoPaint() {
 222   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 223   DCHECK(paint_scheduled_);
 224   paint_scheduled_ = false;
 225
 226   // If the view size is empty or we have no output buffers ready, return.
 227   if (buffers_.empty() || view_size_.is_empty())
 228     return;
 229
 230   // If no Decoder is initialized, or the host dimensions are empty, return.
 231   if (!decoder_.get() || source_size_.is_empty())
 232     return;
 233
 234   // Draw the invalidated region to the buffer.
 235   webrtc::DesktopFrame* buffer = buffers_.front();
 236   webrtc::DesktopRegion output_region;
 237   decoder_->RenderFrame(view_size_, clip_area_,
 238                         buffer->data(), buffer->stride(), &output_region);
 239
 240   // Notify the consumer that painting is done.
 241   if (!output_region.is_empty()) {
 242     buffers_.pop_front();
 243     consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region,
 244                            decoder_->GetImageShape());
 245   }
 246 }
 247
 248 void SoftwareVideoRenderer::Core::RequestReturnBuffers(
 249     const base::Closure& done) {
 250   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 251
 252   while (!buffers_.empty()) {
 253     consumer_->ReturnBuffer(buffers_.front());
 254     buffers_.pop_front();
 255   }
 256
 257   if (!done.is_null())
 258     done.Run();
 259 }
 260
 261 void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) {
 262   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 263   DCHECK(clip_area_.width() <= buffer->size().width() &&
 264          clip_area_.height() <= buffer->size().height());
 265
 266   buffers_.push_back(buffer);
 267   SchedulePaint();
 268 }
 269
 270 void SoftwareVideoRenderer::Core::InvalidateRegion(
 271     const webrtc::DesktopRegion& region) {
 272   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 273
 274   if (decoder_.get()) {
 275     decoder_->Invalidate(view_size_, region);
 276     SchedulePaint();
 277   }
 278 }
 279
 280 void SoftwareVideoRenderer::Core::SetOutputSizeAndClip(
 281     const webrtc::DesktopSize& view_size,
 282     const webrtc::DesktopRect& clip_area) {
 283   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 284
 285   // The whole frame needs to be repainted if the scaling factor has changed.
 286   if (!view_size_.equals(view_size) && decoder_.get()) {
 287     webrtc::DesktopRegion region;
 288     region.AddRect(webrtc::DesktopRect::MakeSize(view_size));
 289     decoder_->Invalidate(view_size, region);
 290   }
 291
 292   if (!view_size_.equals(view_size) ||
 293       !clip_area_.equals(clip_area)) {
 294     view_size_ = view_size;
 295     clip_area_ = clip_area;
 296
 297     // Return buffers that are smaller than needed to the consumer for
 298     // reuse/reallocation.
 299     std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin();
 300     while (i != buffers_.end()) {
 301       if ((*i)->size().width() < clip_area_.width() ||
 302           (*i)->size().height() < clip_area_.height()) {
 303         consumer_->ReturnBuffer(*i);
 304         i = buffers_.erase(i);
 305       } else {
 306         ++i;
 307       }
 308     }
 309
 310     SchedulePaint();
 311   }
 312 }
 313
 314 SoftwareVideoRenderer::SoftwareVideoRenderer(
 315     scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
 316     scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
 317     scoped_ptr<FrameConsumerProxy> consumer)
 318     : decode_task_runner_(decode_task_runner),
 319       core_(new Core(main_task_runner, decode_task_runner, consumer.Pass())),
 320       latest_event_timestamp_(0),
 321       weak_factory_(this) {
 322   DCHECK(CalledOnValidThread());
 323 }
 324
 325 SoftwareVideoRenderer::~SoftwareVideoRenderer() {
 326   DCHECK(CalledOnValidThread());
 327   bool result = decode_task_runner_->DeleteSoon(FROM_HERE, core_.release());
 328   DCHECK(result);
 329 }
 330
 331 void SoftwareVideoRenderer::OnSessionConfig(
 332     const protocol::SessionConfig& config) {
 333   DCHECK(CalledOnValidThread());
 334   decode_task_runner_->PostTask(
 335       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::OnSessionConfig,
 336                             base::Unretained(core_.get()), config));
 337 }
 338
 339 ChromotingStats* SoftwareVideoRenderer::GetStats() {
 340   DCHECK(CalledOnValidThread());
 341   return &stats_;
 342 }
 343
 344 protocol::VideoStub* SoftwareVideoRenderer::GetVideoStub() {
 345   return this;
 346 }
 347
 348 void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 349                                                 const base::Closure& done) {
 350   DCHECK(CalledOnValidThread());
 351
 352   // Record this received packet, even if it is empty.
 353   stats_.video_packet_rate()->Record(1);
 354
 355   // If the video packet is empty then drop it. Empty packets are used to
 356   // maintain activity on the network.
 357   if (!packet->has_data() || packet->data().size() == 0) {
 358     done.Run();
 359     return;
 360   }
 361
 362   // Add one frame to the video frame rate counter.
 363   stats_.video_frame_rate()->Record(1);
 364
 365   // Record other statistics received from host.
 366   stats_.video_bandwidth()->Record(packet->data().size());
 367   if (packet->has_capture_time_ms())
 368     stats_.video_capture_ms()->Record(packet->capture_time_ms());
 369   if (packet->has_encode_time_ms())
 370     stats_.video_encode_ms()->Record(packet->encode_time_ms());
 371   if (packet->has_latest_event_timestamp() &&
 372       packet->latest_event_timestamp() > latest_event_timestamp_) {
 373     latest_event_timestamp_ = packet->latest_event_timestamp();
 374     base::TimeDelta round_trip_latency =
 375         base::Time::Now() -
 376         base::Time::FromInternalValue(packet->latest_event_timestamp());
 377     stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds());
 378   }
 379
 380   // Measure the latency between the last packet being received and presented.
 381   base::Time decode_start = base::Time::Now();
 382
 383   base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone,
 384                                          weak_factory_.GetWeakPtr(),
 385                                          decode_start, done);
 386
 387   decode_task_runner_->PostTask(FROM_HERE, base::Bind(
 388       &SoftwareVideoRenderer::Core::DecodePacket,
 389       base::Unretained(core_.get()), base::Passed(&packet), decode_done));
 390 }
 391
 392 void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) {
 393   decode_task_runner_->PostTask(
 394       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer,
 395                             base::Unretained(core_.get()), buffer));
 396 }
 397
 398 void SoftwareVideoRenderer::InvalidateRegion(
 399     const webrtc::DesktopRegion& region) {
 400   decode_task_runner_->PostTask(
 401       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion,
 402                             base::Unretained(core_.get()), region));
 403 }
 404
 405 void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) {
 406   decode_task_runner_->PostTask(
 407       FROM_HERE,
 408       base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers,
 409                  base::Unretained(core_.get()), done));
 410 }
 411
 412 void SoftwareVideoRenderer::SetOutputSizeAndClip(
 413     const webrtc::DesktopSize& view_size,
 414     const webrtc::DesktopRect& clip_area) {
 415   decode_task_runner_->PostTask(
 416       FROM_HERE,
 417       base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip,
 418                  base::Unretained(core_.get()), view_size, clip_area));
 419 }
 420
 421 void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start,
 422                                           const base::Closure& done) {
 423   DCHECK(CalledOnValidThread());
 424
 425   // Record the latency between the packet being received and presented.
 426   stats_.video_decode_ms()->Record(
 427       (base::Time::Now() - decode_start).InMilliseconds());
 428
 429   done.Run();
 430 }
 431
 432 }  // namespace remoting