content/browser/renderer_host/media/video_capture_device_client.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/video_capture_device_client.h"
   6
   7 #include "base/bind.h"
   8 #include "base/strings/stringprintf.h"
   9 #include "base/trace_event/trace_event.h"
  10 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
  11 #include "content/browser/renderer_host/media/video_capture_controller.h"
  12 #include "content/public/browser/browser_thread.h"
  13 #include "media/base/bind_to_current_loop.h"
  14 #include "media/base/video_capture_types.h"
  15 #include "media/base/video_frame.h"
  16 #include "third_party/libyuv/include/libyuv.h"
  17
  18 using media::VideoCaptureFormat;
  19 using media::VideoFrame;
  20
  21 namespace content {
  22
  23 // Class combining a Client::Buffer interface implementation and a pool buffer
  24 // implementation to guarantee proper cleanup on destruction on our side.
  25 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
  26  public:
  27   AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
  28                     int buffer_id,
  29                     void* data,
  30                     size_t size)
  31       : pool_(pool),
  32         id_(buffer_id),
  33         data_(data),
  34         size_(size) {
  35     DCHECK(pool_.get());
  36   }
  37   int id() const override { return id_; }
  38   void* data() const override { return data_; }
  39   size_t size() const override { return size_; }
  40
  41  private:
  42   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
  43
  44   const scoped_refptr<VideoCaptureBufferPool> pool_;
  45   const int id_;
  46   void* const data_;
  47   const size_t size_;
  48 };
  49
  50 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
  51     const base::WeakPtr<VideoCaptureController>& controller,
  52     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
  53     : controller_(controller),
  54       buffer_pool_(buffer_pool),
  55       last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {}
  56
  57 VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
  58
  59 void VideoCaptureDeviceClient::OnIncomingCapturedData(
  60     const uint8* data,
  61     int length,
  62     const VideoCaptureFormat& frame_format,
  63     int rotation,
  64     const base::TimeTicks& timestamp) {
  65   TRACE_EVENT0("video", "VideoCaptureDeviceClient::OnIncomingCapturedData");
  66
  67   if (last_captured_pixel_format_ != frame_format.pixel_format) {
  68     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
  69                                  frame_format.pixel_format));
  70     last_captured_pixel_format_ = frame_format.pixel_format;
  71   }
  72
  73   if (!frame_format.IsValid())
  74     return;
  75
  76   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
  77   // bit decomposition of {width, height}, grabbing the odd and even parts.
  78   const int chopped_width = frame_format.frame_size.width() & 1;
  79   const int chopped_height = frame_format.frame_size.height() & 1;
  80   const int new_unrotated_width = frame_format.frame_size.width() & ~1;
  81   const int new_unrotated_height = frame_format.frame_size.height() & ~1;
  82
  83   int destination_width = new_unrotated_width;
  84   int destination_height = new_unrotated_height;
  85   if (rotation == 90 || rotation == 270) {
  86     destination_width = new_unrotated_height;
  87     destination_height = new_unrotated_width;
  88   }
  89
  90   DCHECK_EQ(rotation % 90, 0)
  91       << " Rotation must be a multiple of 90, now: " << rotation;
  92   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
  93   if (rotation == 90)
  94     rotation_mode = libyuv::kRotate90;
  95   else if (rotation == 180)
  96     rotation_mode = libyuv::kRotate180;
  97   else if (rotation == 270)
  98     rotation_mode = libyuv::kRotate270;
  99
 100   const gfx::Size dimensions(destination_width, destination_height);
 101   if (!VideoFrame::IsValidConfig(VideoFrame::I420,
 102                                  dimensions,
 103                                  gfx::Rect(dimensions),
 104                                  dimensions)) {
 105     return;
 106   }
 107
 108   scoped_refptr<Buffer> buffer = ReserveOutputBuffer(VideoFrame::I420,
 109                                                      dimensions);
 110   if (!buffer.get())
 111     return;
 112
 113   uint8* const yplane = reinterpret_cast<uint8*>(buffer->data());
 114   uint8* const uplane =
 115       yplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
 116                                                VideoFrame::kYPlane, dimensions);
 117   uint8* const vplane =
 118       uplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
 119                                                VideoFrame::kUPlane, dimensions);
 120   int yplane_stride = dimensions.width();
 121   int uv_plane_stride = yplane_stride / 2;
 122   int crop_x = 0;
 123   int crop_y = 0;
 124   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
 125
 126   bool flip = false;
 127   switch (frame_format.pixel_format) {
 128     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
 129       break;
 130     case media::PIXEL_FORMAT_I420:
 131       DCHECK(!chopped_width && !chopped_height);
 132       origin_colorspace = libyuv::FOURCC_I420;
 133       break;
 134     case media::PIXEL_FORMAT_YV12:
 135       DCHECK(!chopped_width && !chopped_height);
 136       origin_colorspace = libyuv::FOURCC_YV12;
 137       break;
 138     case media::PIXEL_FORMAT_NV12:
 139       DCHECK(!chopped_width && !chopped_height);
 140       origin_colorspace = libyuv::FOURCC_NV12;
 141       break;
 142     case media::PIXEL_FORMAT_NV21:
 143       DCHECK(!chopped_width && !chopped_height);
 144       origin_colorspace = libyuv::FOURCC_NV21;
 145       break;
 146     case media::PIXEL_FORMAT_YUY2:
 147       DCHECK(!chopped_width && !chopped_height);
 148       origin_colorspace = libyuv::FOURCC_YUY2;
 149       break;
 150     case media::PIXEL_FORMAT_UYVY:
 151       DCHECK(!chopped_width && !chopped_height);
 152       origin_colorspace = libyuv::FOURCC_UYVY;
 153       break;
 154     case media::PIXEL_FORMAT_RGB24:
 155       origin_colorspace = libyuv::FOURCC_24BG;
 156 #if defined(OS_WIN)
 157       // TODO(wjia): Currently, for RGB24 on WIN, capture device always
 158       // passes in positive src_width and src_height. Remove this hardcoded
 159       // value when nagative src_height is supported. The negative src_height
 160       // indicates that vertical flipping is needed.
 161       flip = true;
 162 #endif
 163       break;
 164     case media::PIXEL_FORMAT_RGB32:
 165 // Fallback to PIXEL_FORMAT_ARGB setting |flip| in Windows platforms.
 166 #if defined(OS_WIN)
 167       flip = true;
 168 #endif
 169     case media::PIXEL_FORMAT_ARGB:
 170       origin_colorspace = libyuv::FOURCC_ARGB;
 171       break;
 172     case media::PIXEL_FORMAT_MJPEG:
 173       origin_colorspace = libyuv::FOURCC_MJPG;
 174       break;
 175     default:
 176       NOTREACHED();
 177   }
 178
 179   // The input |length| can be greater than the required buffer size because of
 180   // paddings and/or alignments, but it cannot be smaller.
 181   DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize());
 182
 183   if (libyuv::ConvertToI420(data,
 184                             length,
 185                             yplane,
 186                             yplane_stride,
 187                             uplane,
 188                             uv_plane_stride,
 189                             vplane,
 190                             uv_plane_stride,
 191                             crop_x,
 192                             crop_y,
 193                             frame_format.frame_size.width(),
 194                             (flip ? -1 : 1) * frame_format.frame_size.height(),
 195                             new_unrotated_width,
 196                             new_unrotated_height,
 197                             rotation_mode,
 198                             origin_colorspace) != 0) {
 199     DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
 200                   << media::VideoCaptureFormat::PixelFormatToString(
 201                          frame_format.pixel_format);
 202     return;
 203   }
 204   scoped_refptr<VideoFrame> frame =
 205       VideoFrame::WrapExternalPackedMemory(
 206           VideoFrame::I420,
 207           dimensions,
 208           gfx::Rect(dimensions),
 209           dimensions,
 210           yplane,
 211           VideoFrame::AllocationSize(VideoFrame::I420, dimensions),
 212           base::SharedMemory::NULLHandle(),
 213           0,
 214           base::TimeDelta(),
 215           base::Closure());
 216   DCHECK(frame.get());
 217   frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
 218                                frame_format.frame_rate);
 219
 220   BrowserThread::PostTask(
 221       BrowserThread::IO,
 222       FROM_HERE,
 223       base::Bind(
 224           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
 225           controller_,
 226           buffer,
 227           frame,
 228           timestamp));
 229 }
 230
 231 void
 232 VideoCaptureDeviceClient::OnIncomingCapturedYuvData(
 233     const uint8* y_data,
 234     const uint8* u_data,
 235     const uint8* v_data,
 236     size_t y_stride,
 237     size_t u_stride,
 238     size_t v_stride,
 239     const VideoCaptureFormat& frame_format,
 240     int clockwise_rotation,
 241     const base::TimeTicks& timestamp) {
 242   TRACE_EVENT0("video", "VideoCaptureDeviceClient::OnIncomingCapturedYuvData");
 243   DCHECK_EQ(frame_format.pixel_format, media::PIXEL_FORMAT_I420);
 244   DCHECK_EQ(clockwise_rotation, 0) << "Rotation not supported";
 245
 246   scoped_refptr<Buffer> buffer = ReserveOutputBuffer(VideoFrame::I420,
 247                                                      frame_format.frame_size);
 248   if (!buffer.get())
 249     return;
 250
 251   // Blit (copy) here from y,u,v into buffer.data()). Needed so we can return
 252   // the parameter buffer synchronously to the driver.
 253   const size_t y_plane_size = VideoFrame::PlaneAllocationSize(VideoFrame::I420,
 254       VideoFrame::kYPlane, frame_format.frame_size);
 255   const size_t u_plane_size = VideoFrame::PlaneAllocationSize(
 256       VideoFrame::I420, VideoFrame::kUPlane, frame_format.frame_size);
 257   uint8* const dst_y = reinterpret_cast<uint8*>(buffer->data());
 258   uint8* const dst_u = dst_y + y_plane_size;
 259   uint8* const dst_v = dst_u + u_plane_size;
 260
 261   const size_t dst_y_stride = VideoFrame::RowBytes(
 262       VideoFrame::kYPlane, VideoFrame::I420, frame_format.frame_size.width());
 263   const size_t dst_u_stride = VideoFrame::RowBytes(
 264       VideoFrame::kUPlane, VideoFrame::I420, frame_format.frame_size.width());
 265   const size_t dst_v_stride = VideoFrame::RowBytes(
 266       VideoFrame::kVPlane, VideoFrame::I420, frame_format.frame_size.width());
 267   DCHECK_GE(y_stride, dst_y_stride);
 268   DCHECK_GE(u_stride, dst_u_stride);
 269   DCHECK_GE(v_stride, dst_v_stride);
 270
 271   if (libyuv::I420Copy(y_data, y_stride,
 272                        u_data, u_stride,
 273                        v_data, v_stride,
 274                        dst_y, dst_y_stride,
 275                        dst_u, dst_u_stride,
 276                        dst_v, dst_v_stride,
 277                        frame_format.frame_size.width(),
 278                        frame_format.frame_size.height())) {
 279     DLOG(WARNING) << "Failed to copy buffer";
 280     return;
 281   }
 282
 283   scoped_refptr<VideoFrame> video_frame = VideoFrame::WrapExternalYuvData(
 284       VideoFrame::I420, frame_format.frame_size,
 285       gfx::Rect(frame_format.frame_size), frame_format.frame_size, y_stride,
 286       u_stride, v_stride, dst_y, dst_u, dst_v, base::TimeDelta(),
 287       base::Closure());
 288   DCHECK(video_frame.get());
 289   video_frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
 290                                      frame_format.frame_rate);
 291
 292   BrowserThread::PostTask(
 293       BrowserThread::IO,
 294       FROM_HERE,
 295       base::Bind(
 296           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
 297           controller_,
 298           buffer,
 299           video_frame,
 300           timestamp));
 301 };
 302
 303 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 304 VideoCaptureDeviceClient::ReserveOutputBuffer(VideoFrame::Format format,
 305                                               const gfx::Size& dimensions) {
 306   const size_t frame_bytes = VideoFrame::AllocationSize(format, dimensions);
 307   if (format == VideoFrame::NATIVE_TEXTURE) {
 308     DCHECK_EQ(dimensions.width(), 0);
 309     DCHECK_EQ(dimensions.height(), 0);
 310   } else {
 311     DLOG_IF(ERROR, frame_bytes == 0) << "Error calculating allocation size";
 312   }
 313
 314   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
 315   const int buffer_id =
 316       buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
 317   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
 318     return NULL;
 319   void* data;
 320   size_t size;
 321   buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
 322
 323   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
 324       new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size));
 325
 326   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
 327     BrowserThread::PostTask(BrowserThread::IO,
 328         FROM_HERE,
 329         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
 330                    controller_, buffer_id_to_drop));
 331   }
 332
 333   return output_buffer;
 334 }
 335
 336 void
 337 VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
 338     const scoped_refptr<Buffer>& buffer,
 339     const scoped_refptr<VideoFrame>& frame,
 340     const base::TimeTicks& timestamp) {
 341   BrowserThread::PostTask(
 342       BrowserThread::IO,
 343       FROM_HERE,
 344       base::Bind(
 345           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
 346           controller_,
 347           buffer,
 348           frame,
 349           timestamp));
 350 }
 351
 352 void VideoCaptureDeviceClient::OnError(
 353     const std::string& reason) {
 354   const std::string log_message = base::StringPrintf(
 355       "Error on video capture: %s, OS message: %s",
 356       reason.c_str(),
 357       logging::SystemErrorCodeToString(
 358           logging::GetLastSystemErrorCode()).c_str());
 359   DLOG(ERROR) << log_message;
 360   OnLog(log_message);
 361   BrowserThread::PostTask(BrowserThread::IO,
 362       FROM_HERE,
 363       base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
 364 }
 365
 366 void VideoCaptureDeviceClient::OnLog(
 367     const std::string& message) {
 368   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 369                           base::Bind(&VideoCaptureController::DoLogOnIOThread,
 370                                      controller_, message));
 371 }
 372
 373 }  // namespace content