content/renderer/media/video_capture_impl.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Notes about usage of this object by VideoCaptureImplManager.
   6 //
   7 // VideoCaptureImplManager access this object by using a Unretained()
   8 // binding and tasks on the IO thread. It is then important that
   9 // VideoCaptureImpl never post task to itself. All operations must be
  10 // synchronous.
  11
  12 #include "content/renderer/media/video_capture_impl.h"
  13
  14 #include "base/bind.h"
  15 #include "base/stl_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "content/child/child_process.h"
  18 #include "content/common/gpu/client/gpu_memory_buffer_impl.h"
  19 #include "content/common/media/video_capture_messages.h"
  20 #include "media/base/bind_to_current_loop.h"
  21 #include "media/base/limits.h"
  22 #include "media/base/video_frame.h"
  23
  24 namespace content {
  25
  26 namespace {
  27
  28 const int kUndefinedDeviceId = 0;
  29
  30 // This is called on an unknown thread when the VideoFrame destructor executes.
  31 // As of this writing, this callback mechanism is the only interface in
  32 // VideoFrame to provide the final value for |release_sync_point|.
  33 // VideoCaptureImpl::DidFinishConsumingFrame() will read the value saved here,
  34 // and pass it back to the IO thread to pass back to the host via the
  35 // BufferReady IPC.
  36 void SaveReleaseSyncPoint(uint32* storage, uint32 release_sync_point) {
  37   *storage = release_sync_point;
  38 }
  39
  40 }  // namespace
  41
  42 // A holder of a memory-backed buffer and accessors to it.
  43 class VideoCaptureImpl::ClientBuffer
  44     : public base::RefCountedThreadSafe<ClientBuffer> {
  45  public:
  46   ClientBuffer(scoped_ptr<base::SharedMemory> buffer, size_t buffer_size)
  47       : buffer_(buffer.Pass()), buffer_size_(buffer_size) {}
  48
  49   base::SharedMemory* buffer() const { return buffer_.get(); }
  50   size_t buffer_size() const { return buffer_size_; }
  51
  52  private:
  53   friend class base::RefCountedThreadSafe<ClientBuffer>;
  54
  55   virtual ~ClientBuffer() {}
  56
  57   const scoped_ptr<base::SharedMemory> buffer_;
  58   const size_t buffer_size_;
  59
  60   DISALLOW_COPY_AND_ASSIGN(ClientBuffer);
  61 };
  62
  63 // A holder of a GpuMemoryBuffer-backed buffer, Map()ed on ctor and Unmap()ed on
  64 // dtor. Creates and owns GpuMemoryBuffer instances.
  65 class VideoCaptureImpl::ClientBuffer2
  66     : public base::RefCountedThreadSafe<ClientBuffer2> {
  67  public:
  68   ClientBuffer2(
  69       const std::vector<gfx::GpuMemoryBufferHandle>& client_handles,
  70       const gfx::Size& size)
  71       : handles_(client_handles),
  72         size_(size) {
  73     const media::VideoPixelFormat format = media::PIXEL_FORMAT_I420;
  74     for (size_t i = 0; i < handles_.size(); ++i) {
  75       const size_t width = media::VideoFrame::Columns(i, format, size_.width());
  76       const size_t height = media::VideoFrame::Rows(i, format, size_.height());
  77       buffers_.push_back(GpuMemoryBufferImpl::CreateFromHandle(
  78           handles_[i],
  79           gfx::Size(width, height),
  80           gfx::BufferFormat::R_8,
  81           gfx::BufferUsage::MAP,
  82           base::Bind(&ClientBuffer2::DestroyGpuMemoryBuffer,
  83                      base::Unretained(this))));
  84       void* data_ptr = nullptr;
  85       buffers_[i]->Map(&data_ptr);
  86       data_[i] = reinterpret_cast<uint8*>(data_ptr);
  87       strides_[i] = width;
  88     }
  89   }
  90
  91   uint8* data(int plane) const { return data_[plane]; }
  92   int32 stride(int plane) const { return strides_[plane]; }
  93   std::vector<gfx::GpuMemoryBufferHandle> gpu_memory_buffer_handles() {
  94     return handles_;
  95   }
  96
  97  private:
  98   friend class base::RefCountedThreadSafe<ClientBuffer2>;
  99
 100   virtual ~ClientBuffer2() {
 101     for (auto& buffer : buffers_)
 102       buffer->Unmap();
 103   }
 104
 105   void DestroyGpuMemoryBuffer(uint32 sync_point) {}
 106
 107   const std::vector<gfx::GpuMemoryBufferHandle> handles_;
 108   const gfx::Size size_;
 109   ScopedVector<gfx::GpuMemoryBuffer> buffers_;
 110   uint8* data_[media::VideoFrame::kMaxPlanes];
 111   int32 strides_[media::VideoFrame::kMaxPlanes];
 112
 113   DISALLOW_COPY_AND_ASSIGN(ClientBuffer2);
 114 };
 115
 116 VideoCaptureImpl::VideoCaptureImpl(
 117     const media::VideoCaptureSessionId session_id,
 118     VideoCaptureMessageFilter* filter)
 119     : message_filter_(filter),
 120       device_id_(kUndefinedDeviceId),
 121       session_id_(session_id),
 122       state_(VIDEO_CAPTURE_STATE_STOPPED),
 123       weak_factory_(this) {
 124   DCHECK(filter);
 125 }
 126
 127 VideoCaptureImpl::~VideoCaptureImpl() {
 128   DCHECK(io_task_runner_->BelongsToCurrentThread());
 129 }
 130
 131 void VideoCaptureImpl::Init() {
 132   // For creating callbacks in unittest, this class may be constructed from a
 133   // different thread than the IO thread, e.g. wherever unittest runs on.
 134   // Therefore, this function should define the thread ownership.
 135 #if DCHECK_IS_ON()
 136   io_task_runner_ = base::ThreadTaskRunnerHandle::Get();
 137 #endif
 138   message_filter_->AddDelegate(this);
 139 }
 140
 141 void VideoCaptureImpl::DeInit() {
 142   DCHECK(io_task_runner_->BelongsToCurrentThread());
 143   if (state_ == VIDEO_CAPTURE_STATE_STARTED)
 144     Send(new VideoCaptureHostMsg_Stop(device_id_));
 145   message_filter_->RemoveDelegate(this);
 146 }
 147
 148 void VideoCaptureImpl::SuspendCapture(bool suspend) {
 149   DCHECK(io_task_runner_->BelongsToCurrentThread());
 150   Send(suspend ? static_cast<IPC::Message*>(
 151                      new VideoCaptureHostMsg_Pause(device_id_))
 152                : static_cast<IPC::Message*>(new VideoCaptureHostMsg_Resume(
 153                      device_id_, session_id_, client_params_)));
 154 }
 155
 156 void VideoCaptureImpl::StartCapture(
 157     const media::VideoCaptureParams& params,
 158     const VideoCaptureStateUpdateCB& state_update_cb,
 159     const VideoCaptureDeliverFrameCB& deliver_frame_cb) {
 160   DCHECK(io_task_runner_->BelongsToCurrentThread());
 161   if (state_update_cb.is_null() || deliver_frame_cb.is_null()) {
 162     DVLOG(1) << "StartCapture: Passed null callbacks.";
 163     return;
 164   }
 165   if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
 166     state_update_cb.Run(VIDEO_CAPTURE_STATE_ERROR);
 167     return;
 168   }
 169
 170   // Save client info regardless of whether we're actually starting capture
 171   // now or not, in case a delegate hasn't yet been added.
 172   client_params_ = params;
 173   state_update_cb_ = state_update_cb;
 174   deliver_frame_cb_ = deliver_frame_cb;
 175   client_params_.requested_format.frame_rate =
 176       std::min(client_params_.requested_format.frame_rate,
 177                static_cast<float>(media::limits::kMaxFramesPerSecond));
 178
 179   // Postpone starting capture until OnDelegateAdded(). Do this after saving
 180   // |client_params_|, etc. as StartCapture will be called with these saved
 181   // values as soon as a delegate has been added
 182   if (device_id_ == kUndefinedDeviceId) {
 183     DVLOG(1) << "StartCapture: Not starting, device_id_ is undefined.";
 184     return;
 185   }
 186
 187   // Finally, we can start (assuming we haven't started already).
 188   // Notify the client that we have started regardless of |state_|.
 189   state_update_cb.Run(VIDEO_CAPTURE_STATE_STARTED);
 190   if (state_ == VIDEO_CAPTURE_STATE_STARTED)
 191     return;
 192   DVLOG(1) << "StartCapture: starting with resolution "
 193            << client_params_.requested_format.frame_size.ToString();
 194   StartCaptureInternal();
 195 }
 196
 197 void VideoCaptureImpl::StopCapture() {
 198   DCHECK(io_task_runner_->BelongsToCurrentThread());
 199   if (state_ == VIDEO_CAPTURE_STATE_STOPPED || !IsInitialized())
 200     return;
 201
 202   DVLOG(1) << "StopCapture: Stopping capture.";
 203   state_update_cb_.Run(VIDEO_CAPTURE_STATE_STOPPED);
 204   StopDevice();
 205   client_buffers_.clear();
 206   client_buffer2s_.clear();
 207   ResetClient();
 208   weak_factory_.InvalidateWeakPtrs();
 209 }
 210
 211 void VideoCaptureImpl::GetDeviceSupportedFormats(
 212     const VideoCaptureDeviceFormatsCB& callback) {
 213   DCHECK(io_task_runner_->BelongsToCurrentThread());
 214   device_formats_cb_queue_.push_back(callback);
 215   if (device_formats_cb_queue_.size() == 1)
 216     Send(new VideoCaptureHostMsg_GetDeviceSupportedFormats(device_id_,
 217                                                            session_id_));
 218 }
 219
 220 void VideoCaptureImpl::GetDeviceFormatsInUse(
 221     const VideoCaptureDeviceFormatsCB& callback) {
 222   DCHECK(io_task_runner_->BelongsToCurrentThread());
 223   device_formats_in_use_cb_queue_.push_back(callback);
 224   if (device_formats_in_use_cb_queue_.size() == 1)
 225     Send(
 226         new VideoCaptureHostMsg_GetDeviceFormatsInUse(device_id_, session_id_));
 227 }
 228
 229 void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
 230                                        int length,
 231                                        int buffer_id) {
 232   DCHECK(io_task_runner_->BelongsToCurrentThread());
 233
 234   // In case client calls StopCapture before the arrival of created buffer,
 235   // just close this buffer and return.
 236   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
 237     base::SharedMemory::CloseHandle(handle);
 238     return;
 239   }
 240
 241   scoped_ptr<base::SharedMemory> shm(new base::SharedMemory(handle, false));
 242   if (!shm->Map(length)) {
 243     DLOG(ERROR) << "OnBufferCreated: Map failed.";
 244     return;
 245   }
 246   const bool inserted =
 247       client_buffers_.insert(std::make_pair(buffer_id, new ClientBuffer(
 248                                                            shm.Pass(), length)))
 249           .second;
 250   DCHECK(inserted);
 251 }
 252
 253 void VideoCaptureImpl::OnBufferCreated2(
 254     const std::vector<gfx::GpuMemoryBufferHandle>& handles,
 255     const gfx::Size& size,
 256     int buffer_id) {
 257   DCHECK(io_task_runner_->BelongsToCurrentThread());
 258
 259   // In case client calls StopCapture before the arrival of created buffer,
 260   // just close this buffer and return.
 261   if (state_ != VIDEO_CAPTURE_STATE_STARTED)
 262     return;
 263
 264   const bool inserted =
 265       client_buffer2s_.insert(std::make_pair(buffer_id,
 266                                              new ClientBuffer2(handles, size)))
 267           .second;
 268   DCHECK(inserted);
 269 }
 270
 271 void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) {
 272   DCHECK(io_task_runner_->BelongsToCurrentThread());
 273
 274   const auto& cb_iter = client_buffers_.find(buffer_id);
 275   if (cb_iter != client_buffers_.end()) {
 276     DCHECK(!cb_iter->second.get() || cb_iter->second->HasOneRef())
 277         << "Instructed to delete buffer we are still using.";
 278     client_buffers_.erase(cb_iter);
 279   } else {
 280     const auto& cb2_iter = client_buffer2s_.find(buffer_id);
 281     if (cb2_iter != client_buffer2s_.end()) {
 282       DCHECK(!cb2_iter->second.get() || cb2_iter->second->HasOneRef())
 283           << "Instructed to delete buffer we are still using.";
 284       client_buffer2s_.erase(cb2_iter);
 285     }
 286   }
 287 }
 288
 289 void VideoCaptureImpl::OnBufferReceived(
 290     int buffer_id,
 291     base::TimeTicks timestamp,
 292     const base::DictionaryValue& metadata,
 293     media::VideoPixelFormat pixel_format,
 294     media::VideoFrame::StorageType storage_type,
 295     const gfx::Size& coded_size,
 296     const gfx::Rect& visible_rect,
 297     const std::vector<gpu::MailboxHolder>& mailbox_holders) {
 298   DCHECK(io_task_runner_->BelongsToCurrentThread());
 299   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
 300     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0, -1.0));
 301     return;
 302   }
 303   if (first_frame_timestamp_.is_null())
 304     first_frame_timestamp_ = timestamp;
 305
 306   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 307   TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
 308                        TRACE_EVENT_SCOPE_THREAD, "timestamp",
 309                        timestamp.ToInternalValue(), "time_delta",
 310                        (timestamp - first_frame_timestamp_).ToInternalValue());
 311
 312   scoped_refptr<media::VideoFrame> frame;
 313   base::Callback<void(uint32, double)> buffer_finished_callback;
 314   uint32* release_sync_point_storage = new uint32(0);
 315   if (mailbox_holders.empty()) {
 316     DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
 317     const auto& iter = client_buffers_.find(buffer_id);
 318     DCHECK(iter != client_buffers_.end());
 319     const scoped_refptr<ClientBuffer> buffer = iter->second;
 320     frame = media::VideoFrame::WrapExternalSharedMemory(
 321         pixel_format,
 322         coded_size,
 323         visible_rect,
 324         gfx::Size(visible_rect.width(), visible_rect.height()),
 325         reinterpret_cast<uint8*>(buffer->buffer()->memory()),
 326         buffer->buffer_size(),
 327         buffer->buffer()->handle(),
 328         0 /* shared_memory_offset */,
 329         timestamp - first_frame_timestamp_);
 330     buffer_finished_callback = media::BindToCurrentLoop(
 331         base::Bind(&VideoCaptureImpl::OnClientBufferFinished,
 332                    weak_factory_.GetWeakPtr(), buffer_id, buffer));
 333   } else {
 334 #if DCHECK_IS_ON()
 335     for (const auto& mailbox_holder : mailbox_holders)
 336       DCHECK(mailbox_holder.mailbox.Verify());
 337     DCHECK(mailbox_holders.size() == 1u || mailbox_holders.size() == 3u);
 338 #endif
 339
 340     scoped_refptr<ClientBuffer2> buffer;
 341     if (mailbox_holders.size() ==
 342         media::VideoFrame::NumPlanes(media::PIXEL_FORMAT_ARGB)) {
 343       DCHECK_EQ(media::PIXEL_FORMAT_ARGB, pixel_format);
 344       frame = media::VideoFrame::WrapNativeTexture(
 345           pixel_format,
 346           mailbox_holders[0],
 347           base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
 348           coded_size,
 349           gfx::Rect(coded_size),
 350           coded_size,
 351           timestamp - first_frame_timestamp_);
 352     } else if (mailbox_holders.size() ==
 353                media::VideoFrame::NumPlanes(media::PIXEL_FORMAT_I420)) {
 354       DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
 355       const auto& iter = client_buffer2s_.find(buffer_id);
 356       DCHECK(iter != client_buffer2s_.end());
 357       buffer = iter->second;
 358       frame = media::VideoFrame::WrapYUV420NativeTextures(
 359           mailbox_holders[media::VideoFrame::kYPlane],
 360           mailbox_holders[media::VideoFrame::kUPlane],
 361           mailbox_holders[media::VideoFrame::kVPlane],
 362           base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
 363           coded_size,
 364           gfx::Rect(coded_size),
 365           coded_size,
 366           timestamp - first_frame_timestamp_);
 367     }
 368     buffer_finished_callback = media::BindToCurrentLoop(
 369         base::Bind(&VideoCaptureImpl::OnClientBufferFinished2,
 370                    weak_factory_.GetWeakPtr(), buffer_id, buffer));
 371   }
 372   frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME,
 373                                   timestamp);
 374   frame->AddDestructionObserver(
 375       base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame,
 376                  frame->metadata(), release_sync_point_storage,
 377                  buffer_finished_callback));
 378   frame->metadata()->MergeInternalValuesFrom(metadata);
 379   deliver_frame_cb_.Run(frame, timestamp);
 380 }
 381
 382 void VideoCaptureImpl::OnClientBufferFinished(
 383     int buffer_id,
 384     const scoped_refptr<ClientBuffer>& /* ignored_buffer */,
 385     uint32 release_sync_point,
 386     double consumer_resource_utilization) {
 387   DCHECK(io_task_runner_->BelongsToCurrentThread());
 388   Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id,
 389                                            release_sync_point,
 390                                            consumer_resource_utilization));
 391 }
 392 void VideoCaptureImpl::OnClientBufferFinished2(
 393     int buffer_id,
 394     const scoped_refptr<ClientBuffer2>& gpu_memory_buffer /* ignored_buffer */,
 395     uint32 release_sync_point,
 396     double consumer_resource_utilization) {
 397   OnClientBufferFinished(buffer_id, scoped_refptr<ClientBuffer>(),
 398                          release_sync_point, consumer_resource_utilization);
 399 }
 400
 401 void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) {
 402   // TODO(ajose): http://crbug.com/522155 improve this state machine.
 403   DCHECK(io_task_runner_->BelongsToCurrentThread());
 404   state_ = state;
 405
 406   if (state == VIDEO_CAPTURE_STATE_STOPPED) {
 407     DVLOG(1) << "OnStateChanged: stopped!, device_id = " << device_id_;
 408     client_buffers_.clear();
 409     client_buffer2s_.clear();
 410     weak_factory_.InvalidateWeakPtrs();
 411     return;
 412   }
 413   if (state == VIDEO_CAPTURE_STATE_ERROR) {
 414     DVLOG(1) << "OnStateChanged: error!, device_id = " << device_id_;
 415     if (!state_update_cb_.is_null())
 416       state_update_cb_.Run(VIDEO_CAPTURE_STATE_ERROR);
 417     ResetClient();
 418     return;
 419   }
 420   if (state == VIDEO_CAPTURE_STATE_ENDED) {
 421     DVLOG(1) << "OnStateChanged: ended!, device_id = " << device_id_;
 422     // We'll only notify the client that the stream has stopped.
 423     if (!state_update_cb_.is_null())
 424       state_update_cb_.Run(VIDEO_CAPTURE_STATE_STOPPED);
 425     ResetClient();
 426     return;
 427   }
 428   NOTREACHED();
 429 }
 430
 431 void VideoCaptureImpl::OnDeviceSupportedFormatsEnumerated(
 432     const media::VideoCaptureFormats& supported_formats) {
 433   DCHECK(io_task_runner_->BelongsToCurrentThread());
 434   for (size_t i = 0; i < device_formats_cb_queue_.size(); ++i)
 435     device_formats_cb_queue_[i].Run(supported_formats);
 436   device_formats_cb_queue_.clear();
 437 }
 438
 439 void VideoCaptureImpl::OnDeviceFormatsInUseReceived(
 440     const media::VideoCaptureFormats& formats_in_use) {
 441   DCHECK(io_task_runner_->BelongsToCurrentThread());
 442   for (size_t i = 0; i < device_formats_in_use_cb_queue_.size(); ++i)
 443     device_formats_in_use_cb_queue_[i].Run(formats_in_use);
 444   device_formats_in_use_cb_queue_.clear();
 445 }
 446
 447 void VideoCaptureImpl::OnDelegateAdded(int32 device_id) {
 448   DCHECK(io_task_runner_->BelongsToCurrentThread());
 449   DVLOG(1) << "OnDelegateAdded: device_id " << device_id;
 450
 451   device_id_ = device_id;
 452   StartCapture(client_params_, state_update_cb_, deliver_frame_cb_);
 453 }
 454
 455 void VideoCaptureImpl::StopDevice() {
 456   DCHECK(io_task_runner_->BelongsToCurrentThread());
 457
 458   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 459     Send(new VideoCaptureHostMsg_Stop(device_id_));
 460     client_params_.requested_format.frame_size.SetSize(0, 0);
 461   }
 462 }
 463
 464 void VideoCaptureImpl::RestartCapture() {
 465   DCHECK(io_task_runner_->BelongsToCurrentThread());
 466   DCHECK_EQ(state_, VIDEO_CAPTURE_STATE_STOPPED);
 467
 468   DVLOG(1) << "RestartCapture, restarting capture.";
 469   StartCaptureInternal();
 470 }
 471
 472 void VideoCaptureImpl::StartCaptureInternal() {
 473   DCHECK(io_task_runner_->BelongsToCurrentThread());
 474   DCHECK(device_id_);
 475
 476   Send(new VideoCaptureHostMsg_Start(device_id_, session_id_, client_params_));
 477   state_ = VIDEO_CAPTURE_STATE_STARTED;
 478 }
 479
 480 void VideoCaptureImpl::Send(IPC::Message* message) {
 481   DCHECK(io_task_runner_->BelongsToCurrentThread());
 482   message_filter_->Send(message);
 483 }
 484
 485 // static
 486 void VideoCaptureImpl::DidFinishConsumingFrame(
 487     const media::VideoFrameMetadata* metadata,
 488     uint32* release_sync_point_storage,
 489     const base::Callback<void(uint32, double)>& callback_to_io_thread) {
 490   // Note: This function may be called on any thread by the VideoFrame
 491   // destructor.  |metadata| is still valid for read-access at this point.
 492
 493   uint32 release_sync_point = 0u;
 494   if (release_sync_point_storage) {
 495     release_sync_point = *release_sync_point_storage;
 496     delete release_sync_point_storage;
 497   }
 498
 499   double consumer_resource_utilization = -1.0;
 500   if (!metadata->GetDouble(media::VideoFrameMetadata::RESOURCE_UTILIZATION,
 501                            &consumer_resource_utilization)) {
 502     consumer_resource_utilization = -1.0;
 503   }
 504
 505   callback_to_io_thread.Run(release_sync_point, consumer_resource_utilization);
 506 }
 507
 508 bool VideoCaptureImpl::IsInitialized() const {
 509   return !(state_update_cb_.is_null() || deliver_frame_cb_.is_null()) &&
 510          device_id_ != kUndefinedDeviceId;
 511 }
 512
 513 void VideoCaptureImpl::ResetClient() {
 514   client_params_ = media::VideoCaptureParams();
 515   state_update_cb_.Reset();
 516   deliver_frame_cb_.Reset();
 517   first_frame_timestamp_ = base::TimeTicks();
 518   device_id_ = kUndefinedDeviceId;
 519   state_ = VIDEO_CAPTURE_STATE_STOPPED;
 520 }
 521
 522 }  // namespace content