remoting/client/software_video_renderer.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/software_video_renderer.h"
   6
   7 #include <list>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/callback_helpers.h"
  12 #include "base/location.h"
  13 #include "base/logging.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "remoting/base/util.h"
  16 #include "remoting/client/frame_consumer.h"
  17 #include "remoting/codec/video_decoder.h"
  18 #include "remoting/codec/video_decoder_verbatim.h"
  19 #if !defined(MEDIA_DISABLE_LIBVPX)
  20 #include "remoting/codec/video_decoder_vpx.h"
  21 #endif  // !defined(MEDIA_DISABLE_LIBVPX)
  22 #include "remoting/protocol/session_config.h"
  23 #include "third_party/libyuv/include/libyuv/convert_argb.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  25
  26 using base::Passed;
  27 using remoting::protocol::ChannelConfig;
  28 using remoting::protocol::SessionConfig;
  29
  30 namespace remoting {
  31
  32 // This class wraps a VideoDecoder and byte-swaps the pixels for compatibility
  33 // with the android.graphics.Bitmap class.
  34 // TODO(lambroslambrou): Refactor so that the VideoDecoder produces data
  35 // in the right byte-order, instead of swapping it here.
  36 class RgbToBgrVideoDecoderFilter : public VideoDecoder {
  37  public:
  38   RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent)
  39       : parent_(parent.Pass()) {
  40   }
  41
  42   virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE {
  43     parent_->Initialize(screen_size);
  44   }
  45
  46   virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE {
  47     return parent_->DecodePacket(packet);
  48   }
  49
  50   virtual void Invalidate(const webrtc::DesktopSize& view_size,
  51                           const webrtc::DesktopRegion& region) OVERRIDE {
  52     return parent_->Invalidate(view_size, region);
  53   }
  54
  55   virtual void RenderFrame(const webrtc::DesktopSize& view_size,
  56                            const webrtc::DesktopRect& clip_area,
  57                            uint8* image_buffer,
  58                            int image_stride,
  59                            webrtc::DesktopRegion* output_region) OVERRIDE {
  60     parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride,
  61                          output_region);
  62
  63     for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd();
  64          i.Advance()) {
  65       webrtc::DesktopRect rect = i.rect();
  66       uint8* pixels = image_buffer + (rect.top() * image_stride) +
  67         (rect.left() * kBytesPerPixel);
  68       libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride,
  69                          rect.width(), rect.height());
  70     }
  71   }
  72
  73   virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE {
  74     return parent_->GetImageShape();
  75   }
  76
  77  private:
  78   scoped_ptr<VideoDecoder> parent_;
  79 };
  80
  81 class SoftwareVideoRenderer::Core {
  82  public:
  83   Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
  84        scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
  85        scoped_refptr<FrameConsumerProxy> consumer);
  86   ~Core();
  87
  88   void Initialize(const protocol::SessionConfig& config);
  89   void DrawBuffer(webrtc::DesktopFrame* buffer);
  90   void InvalidateRegion(const webrtc::DesktopRegion& region);
  91   void RequestReturnBuffers(const base::Closure& done);
  92   void SetOutputSizeAndClip(
  93       const webrtc::DesktopSize& view_size,
  94       const webrtc::DesktopRect& clip_area);
  95
  96   // Decodes the contents of |packet|. DecodePacket may keep a reference to
  97   // |packet| so the |packet| must remain alive and valid until |done| is
  98   // executed.
  99   void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done);
 100
 101  private:
 102   // Paints the invalidated region to the next available buffer and returns it
 103   // to the consumer.
 104   void SchedulePaint();
 105   void DoPaint();
 106
 107   scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
 108   scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_;
 109   scoped_refptr<FrameConsumerProxy> consumer_;
 110   scoped_ptr<VideoDecoder> decoder_;
 111
 112   // Remote screen size in pixels.
 113   webrtc::DesktopSize source_size_;
 114
 115   // Vertical and horizontal DPI of the remote screen.
 116   webrtc::DesktopVector source_dpi_;
 117
 118   // The current dimensions of the frame consumer view.
 119   webrtc::DesktopSize view_size_;
 120   webrtc::DesktopRect clip_area_;
 121
 122   // The drawing buffers supplied by the frame consumer.
 123   std::list<webrtc::DesktopFrame*> buffers_;
 124
 125   // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint().
 126   bool paint_scheduled_;
 127
 128   base::WeakPtrFactory<Core> weak_factory_;
 129 };
 130
 131 SoftwareVideoRenderer::Core::Core(
 132     scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
 133     scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
 134     scoped_refptr<FrameConsumerProxy> consumer)
 135     : main_task_runner_(main_task_runner),
 136       decode_task_runner_(decode_task_runner),
 137       consumer_(consumer),
 138       paint_scheduled_(false),
 139       weak_factory_(this) {
 140 }
 141
 142 SoftwareVideoRenderer::Core::~Core() {
 143 }
 144
 145 void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) {
 146   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 147
 148   // Initialize decoder based on the selected codec.
 149   ChannelConfig::Codec codec = config.video_config().codec;
 150   if (codec == ChannelConfig::CODEC_VERBATIM) {
 151     decoder_.reset(new VideoDecoderVerbatim());
 152 #if !defined(MEDIA_DISABLE_LIBVPX)
 153   } else if (codec == ChannelConfig::CODEC_VP8) {
 154     decoder_ = VideoDecoderVpx::CreateForVP8();
 155   } else if (codec == ChannelConfig::CODEC_VP9) {
 156     decoder_ = VideoDecoderVpx::CreateForVP9();
 157   } else {
 158 #endif  // !defined(MEDIA_DISABLE_LIBVPX)
 159     NOTREACHED() << "Invalid Encoding found: " << codec;
 160   }
 161
 162   if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) {
 163     scoped_ptr<VideoDecoder> wrapper(
 164         new RgbToBgrVideoDecoderFilter(decoder_.Pass()));
 165     decoder_ = wrapper.Pass();
 166   }
 167 }
 168
 169 void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet,
 170                                                 const base::Closure& done) {
 171   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 172
 173   bool decoder_needs_reset = false;
 174   bool notify_size_or_dpi_change = false;
 175
 176   // If the packet includes screen size or DPI information, store them.
 177   if (packet->format().has_screen_width() &&
 178       packet->format().has_screen_height()) {
 179     webrtc::DesktopSize source_size(packet->format().screen_width(),
 180                                     packet->format().screen_height());
 181     if (!source_size_.equals(source_size)) {
 182       source_size_ = source_size;
 183       decoder_needs_reset = true;
 184       notify_size_or_dpi_change = true;
 185     }
 186   }
 187   if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
 188     webrtc::DesktopVector source_dpi(packet->format().x_dpi(),
 189                                      packet->format().y_dpi());
 190     if (!source_dpi.equals(source_dpi_)) {
 191       source_dpi_ = source_dpi;
 192       notify_size_or_dpi_change = true;
 193     }
 194   }
 195
 196   // If we've never seen a screen size, ignore the packet.
 197   if (source_size_.is_empty()) {
 198     main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
 199     return;
 200   }
 201
 202   if (decoder_needs_reset)
 203     decoder_->Initialize(source_size_);
 204   if (notify_size_or_dpi_change)
 205     consumer_->SetSourceSize(source_size_, source_dpi_);
 206
 207   if (decoder_->DecodePacket(*packet.get())) {
 208     SchedulePaint();
 209   } else {
 210     LOG(ERROR) << "DecodePacket() failed.";
 211   }
 212
 213   main_task_runner_->PostTask(FROM_HERE, base::Bind(done));
 214 }
 215
 216 void SoftwareVideoRenderer::Core::SchedulePaint() {
 217   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 218   if (paint_scheduled_)
 219     return;
 220   paint_scheduled_ = true;
 221   decode_task_runner_->PostTask(
 222       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint,
 223                             weak_factory_.GetWeakPtr()));
 224 }
 225
 226 void SoftwareVideoRenderer::Core::DoPaint() {
 227   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 228   DCHECK(paint_scheduled_);
 229   paint_scheduled_ = false;
 230
 231   // If the view size is empty or we have no output buffers ready, return.
 232   if (buffers_.empty() || view_size_.is_empty())
 233     return;
 234
 235   // If no Decoder is initialized, or the host dimensions are empty, return.
 236   if (!decoder_.get() || source_size_.is_empty())
 237     return;
 238
 239   // Draw the invalidated region to the buffer.
 240   webrtc::DesktopFrame* buffer = buffers_.front();
 241   webrtc::DesktopRegion output_region;
 242   decoder_->RenderFrame(view_size_, clip_area_,
 243                         buffer->data(), buffer->stride(), &output_region);
 244
 245   // Notify the consumer that painting is done.
 246   if (!output_region.is_empty()) {
 247     buffers_.pop_front();
 248     consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region,
 249                            *decoder_->GetImageShape());
 250   }
 251 }
 252
 253 void SoftwareVideoRenderer::Core::RequestReturnBuffers(
 254     const base::Closure& done) {
 255   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 256
 257   while (!buffers_.empty()) {
 258     consumer_->ReturnBuffer(buffers_.front());
 259     buffers_.pop_front();
 260   }
 261
 262   if (!done.is_null())
 263     done.Run();
 264 }
 265
 266 void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) {
 267   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 268   DCHECK(clip_area_.width() <= buffer->size().width() &&
 269          clip_area_.height() <= buffer->size().height());
 270
 271   buffers_.push_back(buffer);
 272   SchedulePaint();
 273 }
 274
 275 void SoftwareVideoRenderer::Core::InvalidateRegion(
 276     const webrtc::DesktopRegion& region) {
 277   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 278
 279   if (decoder_.get()) {
 280     decoder_->Invalidate(view_size_, region);
 281     SchedulePaint();
 282   }
 283 }
 284
 285 void SoftwareVideoRenderer::Core::SetOutputSizeAndClip(
 286     const webrtc::DesktopSize& view_size,
 287     const webrtc::DesktopRect& clip_area) {
 288   DCHECK(decode_task_runner_->BelongsToCurrentThread());
 289
 290   // The whole frame needs to be repainted if the scaling factor has changed.
 291   if (!view_size_.equals(view_size) && decoder_.get()) {
 292     webrtc::DesktopRegion region;
 293     region.AddRect(webrtc::DesktopRect::MakeSize(view_size));
 294     decoder_->Invalidate(view_size, region);
 295   }
 296
 297   if (!view_size_.equals(view_size) ||
 298       !clip_area_.equals(clip_area)) {
 299     view_size_ = view_size;
 300     clip_area_ = clip_area;
 301
 302     // Return buffers that are smaller than needed to the consumer for
 303     // reuse/reallocation.
 304     std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin();
 305     while (i != buffers_.end()) {
 306       if ((*i)->size().width() < clip_area_.width() ||
 307           (*i)->size().height() < clip_area_.height()) {
 308         consumer_->ReturnBuffer(*i);
 309         i = buffers_.erase(i);
 310       } else {
 311         ++i;
 312       }
 313     }
 314
 315     SchedulePaint();
 316   }
 317 }
 318
 319 SoftwareVideoRenderer::SoftwareVideoRenderer(
 320     scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
 321     scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner,
 322     scoped_refptr<FrameConsumerProxy> consumer)
 323     : decode_task_runner_(decode_task_runner),
 324       core_(new Core(main_task_runner, decode_task_runner, consumer)),
 325       latest_sequence_number_(0),
 326       weak_factory_(this) {
 327   DCHECK(CalledOnValidThread());
 328 }
 329
 330 SoftwareVideoRenderer::~SoftwareVideoRenderer() {
 331   DCHECK(CalledOnValidThread());
 332   decode_task_runner_->DeleteSoon(FROM_HERE, core_.release());
 333 }
 334
 335 void SoftwareVideoRenderer::Initialize(
 336     const protocol::SessionConfig& config) {
 337   DCHECK(CalledOnValidThread());
 338   decode_task_runner_->PostTask(
 339       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize,
 340                             base::Unretained(core_.get()), config));
 341 }
 342
 343 ChromotingStats* SoftwareVideoRenderer::GetStats() {
 344   DCHECK(CalledOnValidThread());
 345   return &stats_;
 346 }
 347
 348 void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 349                                                 const base::Closure& done) {
 350   DCHECK(CalledOnValidThread());
 351
 352   // If the video packet is empty then drop it. Empty packets are used to
 353   // maintain activity on the network.
 354   if (!packet->has_data() || packet->data().size() == 0) {
 355     done.Run();
 356     return;
 357   }
 358
 359   // Add one frame to the counter.
 360   stats_.video_frame_rate()->Record(1);
 361
 362   // Record other statistics received from host.
 363   stats_.video_bandwidth()->Record(packet->data().size());
 364   if (packet->has_capture_time_ms())
 365     stats_.video_capture_ms()->Record(packet->capture_time_ms());
 366   if (packet->has_encode_time_ms())
 367     stats_.video_encode_ms()->Record(packet->encode_time_ms());
 368   if (packet->has_client_sequence_number() &&
 369       packet->client_sequence_number() > latest_sequence_number_) {
 370     latest_sequence_number_ = packet->client_sequence_number();
 371     base::TimeDelta round_trip_latency =
 372         base::Time::Now() -
 373         base::Time::FromInternalValue(packet->client_sequence_number());
 374     stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds());
 375   }
 376
 377   // Measure the latency between the last packet being received and presented.
 378   base::Time decode_start = base::Time::Now();
 379
 380   base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone,
 381                                          weak_factory_.GetWeakPtr(),
 382                                          decode_start, done);
 383
 384   decode_task_runner_->PostTask(FROM_HERE, base::Bind(
 385       &SoftwareVideoRenderer::Core::DecodePacket,
 386       base::Unretained(core_.get()), base::Passed(&packet), decode_done));
 387 }
 388
 389 void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) {
 390   decode_task_runner_->PostTask(
 391       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer,
 392                             base::Unretained(core_.get()), buffer));
 393 }
 394
 395 void SoftwareVideoRenderer::InvalidateRegion(
 396     const webrtc::DesktopRegion& region) {
 397   decode_task_runner_->PostTask(
 398       FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion,
 399                             base::Unretained(core_.get()), region));
 400 }
 401
 402 void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) {
 403   decode_task_runner_->PostTask(
 404       FROM_HERE,
 405       base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers,
 406                  base::Unretained(core_.get()), done));
 407 }
 408
 409 void SoftwareVideoRenderer::SetOutputSizeAndClip(
 410     const webrtc::DesktopSize& view_size,
 411     const webrtc::DesktopRect& clip_area) {
 412   decode_task_runner_->PostTask(
 413       FROM_HERE,
 414       base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip,
 415                  base::Unretained(core_.get()), view_size, clip_area));
 416 }
 417
 418 void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start,
 419                                           const base::Closure& done) {
 420   DCHECK(CalledOnValidThread());
 421
 422   // Record the latency between the packet being received and presented.
 423   stats_.video_decode_ms()->Record(
 424       (base::Time::Now() - decode_start).InMilliseconds());
 425
 426   done.Run();
 427 }
 428
 429 }  // namespace remoting